1 //===-- PPCISelLowering.cpp - PPC DAG Lowering Implementation -------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the PPCISelLowering class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "PPCISelLowering.h" 14 #include "MCTargetDesc/PPCPredicates.h" 15 #include "PPC.h" 16 #include "PPCCCState.h" 17 #include "PPCCallingConv.h" 18 #include "PPCFrameLowering.h" 19 #include "PPCInstrInfo.h" 20 #include "PPCMachineFunctionInfo.h" 21 #include "PPCPerfectShuffle.h" 22 #include "PPCRegisterInfo.h" 23 #include "PPCSubtarget.h" 24 #include "PPCTargetMachine.h" 25 #include "llvm/ADT/APFloat.h" 26 #include "llvm/ADT/APInt.h" 27 #include "llvm/ADT/ArrayRef.h" 28 #include "llvm/ADT/DenseMap.h" 29 #include "llvm/ADT/None.h" 30 #include "llvm/ADT/STLExtras.h" 31 #include "llvm/ADT/SmallPtrSet.h" 32 #include "llvm/ADT/SmallSet.h" 33 #include "llvm/ADT/SmallVector.h" 34 #include "llvm/ADT/Statistic.h" 35 #include "llvm/ADT/StringRef.h" 36 #include "llvm/ADT/StringSwitch.h" 37 #include "llvm/CodeGen/CallingConvLower.h" 38 #include "llvm/CodeGen/ISDOpcodes.h" 39 #include "llvm/CodeGen/MachineBasicBlock.h" 40 #include "llvm/CodeGen/MachineFrameInfo.h" 41 #include "llvm/CodeGen/MachineFunction.h" 42 #include "llvm/CodeGen/MachineInstr.h" 43 #include "llvm/CodeGen/MachineInstrBuilder.h" 44 #include "llvm/CodeGen/MachineJumpTableInfo.h" 45 #include "llvm/CodeGen/MachineLoopInfo.h" 46 #include "llvm/CodeGen/MachineMemOperand.h" 47 #include "llvm/CodeGen/MachineModuleInfo.h" 48 #include "llvm/CodeGen/MachineOperand.h" 49 #include "llvm/CodeGen/MachineRegisterInfo.h" 50 #include "llvm/CodeGen/RuntimeLibcalls.h" 51 #include "llvm/CodeGen/SelectionDAG.h" 52 #include "llvm/CodeGen/SelectionDAGNodes.h" 53 #include "llvm/CodeGen/TargetInstrInfo.h" 54 #include "llvm/CodeGen/TargetLowering.h" 55 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 56 #include "llvm/CodeGen/TargetRegisterInfo.h" 57 #include "llvm/CodeGen/ValueTypes.h" 58 #include "llvm/IR/CallingConv.h" 59 #include "llvm/IR/Constant.h" 60 #include "llvm/IR/Constants.h" 61 #include "llvm/IR/DataLayout.h" 62 #include "llvm/IR/DebugLoc.h" 63 #include "llvm/IR/DerivedTypes.h" 64 #include "llvm/IR/Function.h" 65 #include "llvm/IR/GlobalValue.h" 66 #include "llvm/IR/IRBuilder.h" 67 #include "llvm/IR/Instructions.h" 68 #include "llvm/IR/Intrinsics.h" 69 #include "llvm/IR/IntrinsicsPowerPC.h" 70 #include "llvm/IR/Module.h" 71 #include "llvm/IR/Type.h" 72 #include "llvm/IR/Use.h" 73 #include "llvm/IR/Value.h" 74 #include "llvm/MC/MCContext.h" 75 #include "llvm/MC/MCExpr.h" 76 #include "llvm/MC/MCRegisterInfo.h" 77 #include "llvm/MC/MCSectionXCOFF.h" 78 #include "llvm/MC/MCSymbolXCOFF.h" 79 #include "llvm/Support/AtomicOrdering.h" 80 #include "llvm/Support/BranchProbability.h" 81 #include "llvm/Support/Casting.h" 82 #include "llvm/Support/CodeGen.h" 83 #include "llvm/Support/CommandLine.h" 84 #include "llvm/Support/Compiler.h" 85 #include "llvm/Support/Debug.h" 86 #include "llvm/Support/ErrorHandling.h" 87 #include "llvm/Support/Format.h" 88 #include "llvm/Support/KnownBits.h" 89 #include "llvm/Support/MachineValueType.h" 90 #include "llvm/Support/MathExtras.h" 91 #include "llvm/Support/raw_ostream.h" 92 #include "llvm/Target/TargetMachine.h" 93 #include "llvm/Target/TargetOptions.h" 94 #include <algorithm> 95 #include <cassert> 96 #include <cstdint> 97 #include <iterator> 98 #include <list> 99 #include <utility> 100 #include <vector> 101 102 using namespace llvm; 103 104 #define DEBUG_TYPE "ppc-lowering" 105 106 static cl::opt<bool> DisablePPCPreinc("disable-ppc-preinc", 107 cl::desc("disable preincrement load/store generation on PPC"), cl::Hidden); 108 109 static cl::opt<bool> DisableILPPref("disable-ppc-ilp-pref", 110 cl::desc("disable setting the node scheduling preference to ILP on PPC"), cl::Hidden); 111 112 static cl::opt<bool> DisablePPCUnaligned("disable-ppc-unaligned", 113 cl::desc("disable unaligned load/store generation on PPC"), cl::Hidden); 114 115 static cl::opt<bool> DisableSCO("disable-ppc-sco", 116 cl::desc("disable sibling call optimization on ppc"), cl::Hidden); 117 118 static cl::opt<bool> DisableInnermostLoopAlign32("disable-ppc-innermost-loop-align32", 119 cl::desc("don't always align innermost loop to 32 bytes on ppc"), cl::Hidden); 120 121 static cl::opt<bool> UseAbsoluteJumpTables("ppc-use-absolute-jumptables", 122 cl::desc("use absolute jump tables on ppc"), cl::Hidden); 123 124 static cl::opt<bool> EnableQuadwordAtomics( 125 "ppc-quadword-atomics", 126 cl::desc("enable quadword lock-free atomic operations"), cl::init(false), 127 cl::Hidden); 128 129 STATISTIC(NumTailCalls, "Number of tail calls"); 130 STATISTIC(NumSiblingCalls, "Number of sibling calls"); 131 STATISTIC(ShufflesHandledWithVPERM, "Number of shuffles lowered to a VPERM"); 132 STATISTIC(NumDynamicAllocaProbed, "Number of dynamic stack allocation probed"); 133 134 static bool isNByteElemShuffleMask(ShuffleVectorSDNode *, unsigned, int); 135 136 static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl); 137 138 static const char AIXSSPCanaryWordName[] = "__ssp_canary_word"; 139 140 // FIXME: Remove this once the bug has been fixed! 141 extern cl::opt<bool> ANDIGlueBug; 142 143 PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, 144 const PPCSubtarget &STI) 145 : TargetLowering(TM), Subtarget(STI) { 146 // Initialize map that relates the PPC addressing modes to the computed flags 147 // of a load/store instruction. The map is used to determine the optimal 148 // addressing mode when selecting load and stores. 149 initializeAddrModeMap(); 150 // On PPC32/64, arguments smaller than 4/8 bytes are extended, so all 151 // arguments are at least 4/8 bytes aligned. 152 bool isPPC64 = Subtarget.isPPC64(); 153 setMinStackArgumentAlignment(isPPC64 ? Align(8) : Align(4)); 154 155 // Set up the register classes. 156 addRegisterClass(MVT::i32, &PPC::GPRCRegClass); 157 if (!useSoftFloat()) { 158 if (hasSPE()) { 159 addRegisterClass(MVT::f32, &PPC::GPRCRegClass); 160 // EFPU2 APU only supports f32 161 if (!Subtarget.hasEFPU2()) 162 addRegisterClass(MVT::f64, &PPC::SPERCRegClass); 163 } else { 164 addRegisterClass(MVT::f32, &PPC::F4RCRegClass); 165 addRegisterClass(MVT::f64, &PPC::F8RCRegClass); 166 } 167 } 168 169 // Match BITREVERSE to customized fast code sequence in the td file. 170 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); 171 setOperationAction(ISD::BITREVERSE, MVT::i64, Legal); 172 173 // Sub-word ATOMIC_CMP_SWAP need to ensure that the input is zero-extended. 174 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); 175 176 // Custom lower inline assembly to check for special registers. 177 setOperationAction(ISD::INLINEASM, MVT::Other, Custom); 178 setOperationAction(ISD::INLINEASM_BR, MVT::Other, Custom); 179 180 // PowerPC has an i16 but no i8 (or i1) SEXTLOAD. 181 for (MVT VT : MVT::integer_valuetypes()) { 182 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); 183 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Expand); 184 } 185 186 if (Subtarget.isISA3_0()) { 187 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Legal); 188 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Legal); 189 setTruncStoreAction(MVT::f64, MVT::f16, Legal); 190 setTruncStoreAction(MVT::f32, MVT::f16, Legal); 191 } else { 192 // No extending loads from f16 or HW conversions back and forth. 193 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand); 194 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); 195 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand); 196 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand); 197 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand); 198 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand); 199 setTruncStoreAction(MVT::f64, MVT::f16, Expand); 200 setTruncStoreAction(MVT::f32, MVT::f16, Expand); 201 } 202 203 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 204 205 // PowerPC has pre-inc load and store's. 206 setIndexedLoadAction(ISD::PRE_INC, MVT::i1, Legal); 207 setIndexedLoadAction(ISD::PRE_INC, MVT::i8, Legal); 208 setIndexedLoadAction(ISD::PRE_INC, MVT::i16, Legal); 209 setIndexedLoadAction(ISD::PRE_INC, MVT::i32, Legal); 210 setIndexedLoadAction(ISD::PRE_INC, MVT::i64, Legal); 211 setIndexedStoreAction(ISD::PRE_INC, MVT::i1, Legal); 212 setIndexedStoreAction(ISD::PRE_INC, MVT::i8, Legal); 213 setIndexedStoreAction(ISD::PRE_INC, MVT::i16, Legal); 214 setIndexedStoreAction(ISD::PRE_INC, MVT::i32, Legal); 215 setIndexedStoreAction(ISD::PRE_INC, MVT::i64, Legal); 216 if (!Subtarget.hasSPE()) { 217 setIndexedLoadAction(ISD::PRE_INC, MVT::f32, Legal); 218 setIndexedLoadAction(ISD::PRE_INC, MVT::f64, Legal); 219 setIndexedStoreAction(ISD::PRE_INC, MVT::f32, Legal); 220 setIndexedStoreAction(ISD::PRE_INC, MVT::f64, Legal); 221 } 222 223 // PowerPC uses ADDC/ADDE/SUBC/SUBE to propagate carry. 224 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 }; 225 for (MVT VT : ScalarIntVTs) { 226 setOperationAction(ISD::ADDC, VT, Legal); 227 setOperationAction(ISD::ADDE, VT, Legal); 228 setOperationAction(ISD::SUBC, VT, Legal); 229 setOperationAction(ISD::SUBE, VT, Legal); 230 } 231 232 if (Subtarget.useCRBits()) { 233 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 234 235 if (isPPC64 || Subtarget.hasFPCVT()) { 236 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i1, Promote); 237 AddPromotedToType(ISD::STRICT_SINT_TO_FP, MVT::i1, 238 isPPC64 ? MVT::i64 : MVT::i32); 239 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i1, Promote); 240 AddPromotedToType(ISD::STRICT_UINT_TO_FP, MVT::i1, 241 isPPC64 ? MVT::i64 : MVT::i32); 242 243 setOperationAction(ISD::SINT_TO_FP, MVT::i1, Promote); 244 AddPromotedToType (ISD::SINT_TO_FP, MVT::i1, 245 isPPC64 ? MVT::i64 : MVT::i32); 246 setOperationAction(ISD::UINT_TO_FP, MVT::i1, Promote); 247 AddPromotedToType(ISD::UINT_TO_FP, MVT::i1, 248 isPPC64 ? MVT::i64 : MVT::i32); 249 250 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i1, Promote); 251 AddPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::i1, 252 isPPC64 ? MVT::i64 : MVT::i32); 253 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i1, Promote); 254 AddPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::i1, 255 isPPC64 ? MVT::i64 : MVT::i32); 256 257 setOperationAction(ISD::FP_TO_SINT, MVT::i1, Promote); 258 AddPromotedToType(ISD::FP_TO_SINT, MVT::i1, 259 isPPC64 ? MVT::i64 : MVT::i32); 260 setOperationAction(ISD::FP_TO_UINT, MVT::i1, Promote); 261 AddPromotedToType(ISD::FP_TO_UINT, MVT::i1, 262 isPPC64 ? MVT::i64 : MVT::i32); 263 } else { 264 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i1, Custom); 265 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i1, Custom); 266 setOperationAction(ISD::SINT_TO_FP, MVT::i1, Custom); 267 setOperationAction(ISD::UINT_TO_FP, MVT::i1, Custom); 268 } 269 270 // PowerPC does not support direct load/store of condition registers. 271 setOperationAction(ISD::LOAD, MVT::i1, Custom); 272 setOperationAction(ISD::STORE, MVT::i1, Custom); 273 274 // FIXME: Remove this once the ANDI glue bug is fixed: 275 if (ANDIGlueBug) 276 setOperationAction(ISD::TRUNCATE, MVT::i1, Custom); 277 278 for (MVT VT : MVT::integer_valuetypes()) { 279 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); 280 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote); 281 setTruncStoreAction(VT, MVT::i1, Expand); 282 } 283 284 addRegisterClass(MVT::i1, &PPC::CRBITRCRegClass); 285 } 286 287 // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on 288 // PPC (the libcall is not available). 289 setOperationAction(ISD::FP_TO_SINT, MVT::ppcf128, Custom); 290 setOperationAction(ISD::FP_TO_UINT, MVT::ppcf128, Custom); 291 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::ppcf128, Custom); 292 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::ppcf128, Custom); 293 294 // We do not currently implement these libm ops for PowerPC. 295 setOperationAction(ISD::FFLOOR, MVT::ppcf128, Expand); 296 setOperationAction(ISD::FCEIL, MVT::ppcf128, Expand); 297 setOperationAction(ISD::FTRUNC, MVT::ppcf128, Expand); 298 setOperationAction(ISD::FRINT, MVT::ppcf128, Expand); 299 setOperationAction(ISD::FNEARBYINT, MVT::ppcf128, Expand); 300 setOperationAction(ISD::FREM, MVT::ppcf128, Expand); 301 302 // PowerPC has no SREM/UREM instructions unless we are on P9 303 // On P9 we may use a hardware instruction to compute the remainder. 304 // When the result of both the remainder and the division is required it is 305 // more efficient to compute the remainder from the result of the division 306 // rather than use the remainder instruction. The instructions are legalized 307 // directly because the DivRemPairsPass performs the transformation at the IR 308 // level. 309 if (Subtarget.isISA3_0()) { 310 setOperationAction(ISD::SREM, MVT::i32, Legal); 311 setOperationAction(ISD::UREM, MVT::i32, Legal); 312 setOperationAction(ISD::SREM, MVT::i64, Legal); 313 setOperationAction(ISD::UREM, MVT::i64, Legal); 314 } else { 315 setOperationAction(ISD::SREM, MVT::i32, Expand); 316 setOperationAction(ISD::UREM, MVT::i32, Expand); 317 setOperationAction(ISD::SREM, MVT::i64, Expand); 318 setOperationAction(ISD::UREM, MVT::i64, Expand); 319 } 320 321 // Don't use SMUL_LOHI/UMUL_LOHI or SDIVREM/UDIVREM to lower SREM/UREM. 322 setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand); 323 setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand); 324 setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand); 325 setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand); 326 setOperationAction(ISD::UDIVREM, MVT::i32, Expand); 327 setOperationAction(ISD::SDIVREM, MVT::i32, Expand); 328 setOperationAction(ISD::UDIVREM, MVT::i64, Expand); 329 setOperationAction(ISD::SDIVREM, MVT::i64, Expand); 330 331 // Handle constrained floating-point operations of scalar. 332 // TODO: Handle SPE specific operation. 333 setOperationAction(ISD::STRICT_FADD, MVT::f32, Legal); 334 setOperationAction(ISD::STRICT_FSUB, MVT::f32, Legal); 335 setOperationAction(ISD::STRICT_FMUL, MVT::f32, Legal); 336 setOperationAction(ISD::STRICT_FDIV, MVT::f32, Legal); 337 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal); 338 339 setOperationAction(ISD::STRICT_FADD, MVT::f64, Legal); 340 setOperationAction(ISD::STRICT_FSUB, MVT::f64, Legal); 341 setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal); 342 setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal); 343 344 if (!Subtarget.hasSPE()) { 345 setOperationAction(ISD::STRICT_FMA, MVT::f32, Legal); 346 setOperationAction(ISD::STRICT_FMA, MVT::f64, Legal); 347 } 348 349 if (Subtarget.hasVSX()) { 350 setOperationAction(ISD::STRICT_FRINT, MVT::f32, Legal); 351 setOperationAction(ISD::STRICT_FRINT, MVT::f64, Legal); 352 } 353 354 if (Subtarget.hasFSQRT()) { 355 setOperationAction(ISD::STRICT_FSQRT, MVT::f32, Legal); 356 setOperationAction(ISD::STRICT_FSQRT, MVT::f64, Legal); 357 } 358 359 if (Subtarget.hasFPRND()) { 360 setOperationAction(ISD::STRICT_FFLOOR, MVT::f32, Legal); 361 setOperationAction(ISD::STRICT_FCEIL, MVT::f32, Legal); 362 setOperationAction(ISD::STRICT_FTRUNC, MVT::f32, Legal); 363 setOperationAction(ISD::STRICT_FROUND, MVT::f32, Legal); 364 365 setOperationAction(ISD::STRICT_FFLOOR, MVT::f64, Legal); 366 setOperationAction(ISD::STRICT_FCEIL, MVT::f64, Legal); 367 setOperationAction(ISD::STRICT_FTRUNC, MVT::f64, Legal); 368 setOperationAction(ISD::STRICT_FROUND, MVT::f64, Legal); 369 } 370 371 // We don't support sin/cos/sqrt/fmod/pow 372 setOperationAction(ISD::FSIN , MVT::f64, Expand); 373 setOperationAction(ISD::FCOS , MVT::f64, Expand); 374 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); 375 setOperationAction(ISD::FREM , MVT::f64, Expand); 376 setOperationAction(ISD::FPOW , MVT::f64, Expand); 377 setOperationAction(ISD::FSIN , MVT::f32, Expand); 378 setOperationAction(ISD::FCOS , MVT::f32, Expand); 379 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); 380 setOperationAction(ISD::FREM , MVT::f32, Expand); 381 setOperationAction(ISD::FPOW , MVT::f32, Expand); 382 if (Subtarget.hasSPE()) { 383 setOperationAction(ISD::FMA , MVT::f64, Expand); 384 setOperationAction(ISD::FMA , MVT::f32, Expand); 385 } else { 386 setOperationAction(ISD::FMA , MVT::f64, Legal); 387 setOperationAction(ISD::FMA , MVT::f32, Legal); 388 } 389 390 if (Subtarget.hasSPE()) 391 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand); 392 393 setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom); 394 395 // If we're enabling GP optimizations, use hardware square root 396 if (!Subtarget.hasFSQRT() && 397 !(TM.Options.UnsafeFPMath && Subtarget.hasFRSQRTE() && 398 Subtarget.hasFRE())) 399 setOperationAction(ISD::FSQRT, MVT::f64, Expand); 400 401 if (!Subtarget.hasFSQRT() && 402 !(TM.Options.UnsafeFPMath && Subtarget.hasFRSQRTES() && 403 Subtarget.hasFRES())) 404 setOperationAction(ISD::FSQRT, MVT::f32, Expand); 405 406 if (Subtarget.hasFCPSGN()) { 407 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Legal); 408 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Legal); 409 } else { 410 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 411 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); 412 } 413 414 if (Subtarget.hasFPRND()) { 415 setOperationAction(ISD::FFLOOR, MVT::f64, Legal); 416 setOperationAction(ISD::FCEIL, MVT::f64, Legal); 417 setOperationAction(ISD::FTRUNC, MVT::f64, Legal); 418 setOperationAction(ISD::FROUND, MVT::f64, Legal); 419 420 setOperationAction(ISD::FFLOOR, MVT::f32, Legal); 421 setOperationAction(ISD::FCEIL, MVT::f32, Legal); 422 setOperationAction(ISD::FTRUNC, MVT::f32, Legal); 423 setOperationAction(ISD::FROUND, MVT::f32, Legal); 424 } 425 426 // PowerPC does not have BSWAP, but we can use vector BSWAP instruction xxbrd 427 // to speed up scalar BSWAP64. 428 // CTPOP or CTTZ were introduced in P8/P9 respectively 429 setOperationAction(ISD::BSWAP, MVT::i32 , Expand); 430 if (Subtarget.hasP9Vector() && Subtarget.isPPC64()) 431 setOperationAction(ISD::BSWAP, MVT::i64 , Custom); 432 else 433 setOperationAction(ISD::BSWAP, MVT::i64 , Expand); 434 if (Subtarget.isISA3_0()) { 435 setOperationAction(ISD::CTTZ , MVT::i32 , Legal); 436 setOperationAction(ISD::CTTZ , MVT::i64 , Legal); 437 } else { 438 setOperationAction(ISD::CTTZ , MVT::i32 , Expand); 439 setOperationAction(ISD::CTTZ , MVT::i64 , Expand); 440 } 441 442 if (Subtarget.hasPOPCNTD() == PPCSubtarget::POPCNTD_Fast) { 443 setOperationAction(ISD::CTPOP, MVT::i32 , Legal); 444 setOperationAction(ISD::CTPOP, MVT::i64 , Legal); 445 } else { 446 setOperationAction(ISD::CTPOP, MVT::i32 , Expand); 447 setOperationAction(ISD::CTPOP, MVT::i64 , Expand); 448 } 449 450 // PowerPC does not have ROTR 451 setOperationAction(ISD::ROTR, MVT::i32 , Expand); 452 setOperationAction(ISD::ROTR, MVT::i64 , Expand); 453 454 if (!Subtarget.useCRBits()) { 455 // PowerPC does not have Select 456 setOperationAction(ISD::SELECT, MVT::i32, Expand); 457 setOperationAction(ISD::SELECT, MVT::i64, Expand); 458 setOperationAction(ISD::SELECT, MVT::f32, Expand); 459 setOperationAction(ISD::SELECT, MVT::f64, Expand); 460 } 461 462 // PowerPC wants to turn select_cc of FP into fsel when possible. 463 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); 464 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom); 465 466 // PowerPC wants to optimize integer setcc a bit 467 if (!Subtarget.useCRBits()) 468 setOperationAction(ISD::SETCC, MVT::i32, Custom); 469 470 if (Subtarget.hasFPU()) { 471 setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Legal); 472 setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Legal); 473 setOperationAction(ISD::STRICT_FSETCC, MVT::f128, Legal); 474 475 setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Legal); 476 setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Legal); 477 setOperationAction(ISD::STRICT_FSETCCS, MVT::f128, Legal); 478 } 479 480 // PowerPC does not have BRCOND which requires SetCC 481 if (!Subtarget.useCRBits()) 482 setOperationAction(ISD::BRCOND, MVT::Other, Expand); 483 484 setOperationAction(ISD::BR_JT, MVT::Other, Expand); 485 486 if (Subtarget.hasSPE()) { 487 // SPE has built-in conversions 488 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Legal); 489 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Legal); 490 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Legal); 491 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Legal); 492 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Legal); 493 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Legal); 494 495 // SPE supports signaling compare of f32/f64. 496 setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Legal); 497 setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Legal); 498 } else { 499 // PowerPC turns FP_TO_SINT into FCTIWZ and some load/stores. 500 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom); 501 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); 502 503 // PowerPC does not have [U|S]INT_TO_FP 504 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Expand); 505 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Expand); 506 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Expand); 507 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Expand); 508 } 509 510 if (Subtarget.hasDirectMove() && isPPC64) { 511 setOperationAction(ISD::BITCAST, MVT::f32, Legal); 512 setOperationAction(ISD::BITCAST, MVT::i32, Legal); 513 setOperationAction(ISD::BITCAST, MVT::i64, Legal); 514 setOperationAction(ISD::BITCAST, MVT::f64, Legal); 515 if (TM.Options.UnsafeFPMath) { 516 setOperationAction(ISD::LRINT, MVT::f64, Legal); 517 setOperationAction(ISD::LRINT, MVT::f32, Legal); 518 setOperationAction(ISD::LLRINT, MVT::f64, Legal); 519 setOperationAction(ISD::LLRINT, MVT::f32, Legal); 520 setOperationAction(ISD::LROUND, MVT::f64, Legal); 521 setOperationAction(ISD::LROUND, MVT::f32, Legal); 522 setOperationAction(ISD::LLROUND, MVT::f64, Legal); 523 setOperationAction(ISD::LLROUND, MVT::f32, Legal); 524 } 525 } else { 526 setOperationAction(ISD::BITCAST, MVT::f32, Expand); 527 setOperationAction(ISD::BITCAST, MVT::i32, Expand); 528 setOperationAction(ISD::BITCAST, MVT::i64, Expand); 529 setOperationAction(ISD::BITCAST, MVT::f64, Expand); 530 } 531 532 // We cannot sextinreg(i1). Expand to shifts. 533 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 534 535 // NOTE: EH_SJLJ_SETJMP/_LONGJMP supported here is NOT intended to support 536 // SjLj exception handling but a light-weight setjmp/longjmp replacement to 537 // support continuation, user-level threading, and etc.. As a result, no 538 // other SjLj exception interfaces are implemented and please don't build 539 // your own exception handling based on them. 540 // LLVM/Clang supports zero-cost DWARF exception handling. 541 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom); 542 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom); 543 544 // We want to legalize GlobalAddress and ConstantPool nodes into the 545 // appropriate instructions to materialize the address. 546 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); 547 setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom); 548 setOperationAction(ISD::BlockAddress, MVT::i32, Custom); 549 setOperationAction(ISD::ConstantPool, MVT::i32, Custom); 550 setOperationAction(ISD::JumpTable, MVT::i32, Custom); 551 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); 552 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); 553 setOperationAction(ISD::BlockAddress, MVT::i64, Custom); 554 setOperationAction(ISD::ConstantPool, MVT::i64, Custom); 555 setOperationAction(ISD::JumpTable, MVT::i64, Custom); 556 557 // TRAP is legal. 558 setOperationAction(ISD::TRAP, MVT::Other, Legal); 559 560 // TRAMPOLINE is custom lowered. 561 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom); 562 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom); 563 564 // VASTART needs to be custom lowered to use the VarArgsFrameIndex 565 setOperationAction(ISD::VASTART , MVT::Other, Custom); 566 567 if (Subtarget.is64BitELFABI()) { 568 // VAARG always uses double-word chunks, so promote anything smaller. 569 setOperationAction(ISD::VAARG, MVT::i1, Promote); 570 AddPromotedToType(ISD::VAARG, MVT::i1, MVT::i64); 571 setOperationAction(ISD::VAARG, MVT::i8, Promote); 572 AddPromotedToType(ISD::VAARG, MVT::i8, MVT::i64); 573 setOperationAction(ISD::VAARG, MVT::i16, Promote); 574 AddPromotedToType(ISD::VAARG, MVT::i16, MVT::i64); 575 setOperationAction(ISD::VAARG, MVT::i32, Promote); 576 AddPromotedToType(ISD::VAARG, MVT::i32, MVT::i64); 577 setOperationAction(ISD::VAARG, MVT::Other, Expand); 578 } else if (Subtarget.is32BitELFABI()) { 579 // VAARG is custom lowered with the 32-bit SVR4 ABI. 580 setOperationAction(ISD::VAARG, MVT::Other, Custom); 581 setOperationAction(ISD::VAARG, MVT::i64, Custom); 582 } else 583 setOperationAction(ISD::VAARG, MVT::Other, Expand); 584 585 // VACOPY is custom lowered with the 32-bit SVR4 ABI. 586 if (Subtarget.is32BitELFABI()) 587 setOperationAction(ISD::VACOPY , MVT::Other, Custom); 588 else 589 setOperationAction(ISD::VACOPY , MVT::Other, Expand); 590 591 // Use the default implementation. 592 setOperationAction(ISD::VAEND , MVT::Other, Expand); 593 setOperationAction(ISD::STACKSAVE , MVT::Other, Expand); 594 setOperationAction(ISD::STACKRESTORE , MVT::Other, Custom); 595 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32 , Custom); 596 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64 , Custom); 597 setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, MVT::i32, Custom); 598 setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, MVT::i64, Custom); 599 setOperationAction(ISD::EH_DWARF_CFA, MVT::i32, Custom); 600 setOperationAction(ISD::EH_DWARF_CFA, MVT::i64, Custom); 601 602 // We want to custom lower some of our intrinsics. 603 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 604 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::f64, Custom); 605 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::ppcf128, Custom); 606 607 // To handle counter-based loop conditions. 608 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i1, Custom); 609 610 setOperationAction(ISD::INTRINSIC_VOID, MVT::i8, Custom); 611 setOperationAction(ISD::INTRINSIC_VOID, MVT::i16, Custom); 612 setOperationAction(ISD::INTRINSIC_VOID, MVT::i32, Custom); 613 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); 614 615 // Comparisons that require checking two conditions. 616 if (Subtarget.hasSPE()) { 617 setCondCodeAction(ISD::SETO, MVT::f32, Expand); 618 setCondCodeAction(ISD::SETO, MVT::f64, Expand); 619 setCondCodeAction(ISD::SETUO, MVT::f32, Expand); 620 setCondCodeAction(ISD::SETUO, MVT::f64, Expand); 621 } 622 setCondCodeAction(ISD::SETULT, MVT::f32, Expand); 623 setCondCodeAction(ISD::SETULT, MVT::f64, Expand); 624 setCondCodeAction(ISD::SETUGT, MVT::f32, Expand); 625 setCondCodeAction(ISD::SETUGT, MVT::f64, Expand); 626 setCondCodeAction(ISD::SETUEQ, MVT::f32, Expand); 627 setCondCodeAction(ISD::SETUEQ, MVT::f64, Expand); 628 setCondCodeAction(ISD::SETOGE, MVT::f32, Expand); 629 setCondCodeAction(ISD::SETOGE, MVT::f64, Expand); 630 setCondCodeAction(ISD::SETOLE, MVT::f32, Expand); 631 setCondCodeAction(ISD::SETOLE, MVT::f64, Expand); 632 setCondCodeAction(ISD::SETONE, MVT::f32, Expand); 633 setCondCodeAction(ISD::SETONE, MVT::f64, Expand); 634 635 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal); 636 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal); 637 638 if (Subtarget.has64BitSupport()) { 639 // They also have instructions for converting between i64 and fp. 640 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom); 641 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Expand); 642 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom); 643 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Expand); 644 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); 645 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Expand); 646 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); 647 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand); 648 // This is just the low 32 bits of a (signed) fp->i64 conversion. 649 // We cannot do this with Promote because i64 is not a legal type. 650 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom); 651 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); 652 653 if (Subtarget.hasLFIWAX() || Subtarget.isPPC64()) { 654 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); 655 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom); 656 } 657 } else { 658 // PowerPC does not have FP_TO_UINT on 32-bit implementations. 659 if (Subtarget.hasSPE()) { 660 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Legal); 661 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Legal); 662 } else { 663 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Expand); 664 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Expand); 665 } 666 } 667 668 // With the instructions enabled under FPCVT, we can do everything. 669 if (Subtarget.hasFPCVT()) { 670 if (Subtarget.has64BitSupport()) { 671 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom); 672 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom); 673 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom); 674 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom); 675 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); 676 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); 677 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); 678 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom); 679 } 680 681 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom); 682 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom); 683 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom); 684 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom); 685 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); 686 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); 687 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); 688 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom); 689 } 690 691 if (Subtarget.use64BitRegs()) { 692 // 64-bit PowerPC implementations can support i64 types directly 693 addRegisterClass(MVT::i64, &PPC::G8RCRegClass); 694 // BUILD_PAIR can't be handled natively, and should be expanded to shl/or 695 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand); 696 // 64-bit PowerPC wants to expand i128 shifts itself. 697 setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom); 698 setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom); 699 setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom); 700 } else { 701 // 32-bit PowerPC wants to expand i64 shifts itself. 702 setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom); 703 setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom); 704 setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom); 705 } 706 707 // PowerPC has better expansions for funnel shifts than the generic 708 // TargetLowering::expandFunnelShift. 709 if (Subtarget.has64BitSupport()) { 710 setOperationAction(ISD::FSHL, MVT::i64, Custom); 711 setOperationAction(ISD::FSHR, MVT::i64, Custom); 712 } 713 setOperationAction(ISD::FSHL, MVT::i32, Custom); 714 setOperationAction(ISD::FSHR, MVT::i32, Custom); 715 716 if (Subtarget.hasVSX()) { 717 setOperationAction(ISD::FMAXNUM_IEEE, MVT::f64, Legal); 718 setOperationAction(ISD::FMAXNUM_IEEE, MVT::f32, Legal); 719 setOperationAction(ISD::FMINNUM_IEEE, MVT::f64, Legal); 720 setOperationAction(ISD::FMINNUM_IEEE, MVT::f32, Legal); 721 } 722 723 if (Subtarget.hasAltivec()) { 724 for (MVT VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) { 725 setOperationAction(ISD::SADDSAT, VT, Legal); 726 setOperationAction(ISD::SSUBSAT, VT, Legal); 727 setOperationAction(ISD::UADDSAT, VT, Legal); 728 setOperationAction(ISD::USUBSAT, VT, Legal); 729 } 730 // First set operation action for all vector types to expand. Then we 731 // will selectively turn on ones that can be effectively codegen'd. 732 for (MVT VT : MVT::fixedlen_vector_valuetypes()) { 733 // add/sub are legal for all supported vector VT's. 734 setOperationAction(ISD::ADD, VT, Legal); 735 setOperationAction(ISD::SUB, VT, Legal); 736 737 // For v2i64, these are only valid with P8Vector. This is corrected after 738 // the loop. 739 if (VT.getSizeInBits() <= 128 && VT.getScalarSizeInBits() <= 64) { 740 setOperationAction(ISD::SMAX, VT, Legal); 741 setOperationAction(ISD::SMIN, VT, Legal); 742 setOperationAction(ISD::UMAX, VT, Legal); 743 setOperationAction(ISD::UMIN, VT, Legal); 744 } 745 else { 746 setOperationAction(ISD::SMAX, VT, Expand); 747 setOperationAction(ISD::SMIN, VT, Expand); 748 setOperationAction(ISD::UMAX, VT, Expand); 749 setOperationAction(ISD::UMIN, VT, Expand); 750 } 751 752 if (Subtarget.hasVSX()) { 753 setOperationAction(ISD::FMAXNUM, VT, Legal); 754 setOperationAction(ISD::FMINNUM, VT, Legal); 755 } 756 757 // Vector instructions introduced in P8 758 if (Subtarget.hasP8Altivec() && (VT.SimpleTy != MVT::v1i128)) { 759 setOperationAction(ISD::CTPOP, VT, Legal); 760 setOperationAction(ISD::CTLZ, VT, Legal); 761 } 762 else { 763 setOperationAction(ISD::CTPOP, VT, Expand); 764 setOperationAction(ISD::CTLZ, VT, Expand); 765 } 766 767 // Vector instructions introduced in P9 768 if (Subtarget.hasP9Altivec() && (VT.SimpleTy != MVT::v1i128)) 769 setOperationAction(ISD::CTTZ, VT, Legal); 770 else 771 setOperationAction(ISD::CTTZ, VT, Expand); 772 773 // We promote all shuffles to v16i8. 774 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Promote); 775 AddPromotedToType (ISD::VECTOR_SHUFFLE, VT, MVT::v16i8); 776 777 // We promote all non-typed operations to v4i32. 778 setOperationAction(ISD::AND , VT, Promote); 779 AddPromotedToType (ISD::AND , VT, MVT::v4i32); 780 setOperationAction(ISD::OR , VT, Promote); 781 AddPromotedToType (ISD::OR , VT, MVT::v4i32); 782 setOperationAction(ISD::XOR , VT, Promote); 783 AddPromotedToType (ISD::XOR , VT, MVT::v4i32); 784 setOperationAction(ISD::LOAD , VT, Promote); 785 AddPromotedToType (ISD::LOAD , VT, MVT::v4i32); 786 setOperationAction(ISD::SELECT, VT, Promote); 787 AddPromotedToType (ISD::SELECT, VT, MVT::v4i32); 788 setOperationAction(ISD::VSELECT, VT, Legal); 789 setOperationAction(ISD::SELECT_CC, VT, Promote); 790 AddPromotedToType (ISD::SELECT_CC, VT, MVT::v4i32); 791 setOperationAction(ISD::STORE, VT, Promote); 792 AddPromotedToType (ISD::STORE, VT, MVT::v4i32); 793 794 // No other operations are legal. 795 setOperationAction(ISD::MUL , VT, Expand); 796 setOperationAction(ISD::SDIV, VT, Expand); 797 setOperationAction(ISD::SREM, VT, Expand); 798 setOperationAction(ISD::UDIV, VT, Expand); 799 setOperationAction(ISD::UREM, VT, Expand); 800 setOperationAction(ISD::FDIV, VT, Expand); 801 setOperationAction(ISD::FREM, VT, Expand); 802 setOperationAction(ISD::FNEG, VT, Expand); 803 setOperationAction(ISD::FSQRT, VT, Expand); 804 setOperationAction(ISD::FLOG, VT, Expand); 805 setOperationAction(ISD::FLOG10, VT, Expand); 806 setOperationAction(ISD::FLOG2, VT, Expand); 807 setOperationAction(ISD::FEXP, VT, Expand); 808 setOperationAction(ISD::FEXP2, VT, Expand); 809 setOperationAction(ISD::FSIN, VT, Expand); 810 setOperationAction(ISD::FCOS, VT, Expand); 811 setOperationAction(ISD::FABS, VT, Expand); 812 setOperationAction(ISD::FFLOOR, VT, Expand); 813 setOperationAction(ISD::FCEIL, VT, Expand); 814 setOperationAction(ISD::FTRUNC, VT, Expand); 815 setOperationAction(ISD::FRINT, VT, Expand); 816 setOperationAction(ISD::FNEARBYINT, VT, Expand); 817 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Expand); 818 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand); 819 setOperationAction(ISD::BUILD_VECTOR, VT, Expand); 820 setOperationAction(ISD::MULHU, VT, Expand); 821 setOperationAction(ISD::MULHS, VT, Expand); 822 setOperationAction(ISD::UMUL_LOHI, VT, Expand); 823 setOperationAction(ISD::SMUL_LOHI, VT, Expand); 824 setOperationAction(ISD::UDIVREM, VT, Expand); 825 setOperationAction(ISD::SDIVREM, VT, Expand); 826 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Expand); 827 setOperationAction(ISD::FPOW, VT, Expand); 828 setOperationAction(ISD::BSWAP, VT, Expand); 829 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); 830 setOperationAction(ISD::ROTL, VT, Expand); 831 setOperationAction(ISD::ROTR, VT, Expand); 832 833 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) { 834 setTruncStoreAction(VT, InnerVT, Expand); 835 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); 836 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); 837 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); 838 } 839 } 840 setOperationAction(ISD::SELECT_CC, MVT::v4i32, Expand); 841 if (!Subtarget.hasP8Vector()) { 842 setOperationAction(ISD::SMAX, MVT::v2i64, Expand); 843 setOperationAction(ISD::SMIN, MVT::v2i64, Expand); 844 setOperationAction(ISD::UMAX, MVT::v2i64, Expand); 845 setOperationAction(ISD::UMIN, MVT::v2i64, Expand); 846 } 847 848 // We can custom expand all VECTOR_SHUFFLEs to VPERM, others we can handle 849 // with merges, splats, etc. 850 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16i8, Custom); 851 852 // Vector truncates to sub-word integer that fit in an Altivec/VSX register 853 // are cheap, so handle them before they get expanded to scalar. 854 setOperationAction(ISD::TRUNCATE, MVT::v8i8, Custom); 855 setOperationAction(ISD::TRUNCATE, MVT::v4i8, Custom); 856 setOperationAction(ISD::TRUNCATE, MVT::v2i8, Custom); 857 setOperationAction(ISD::TRUNCATE, MVT::v4i16, Custom); 858 setOperationAction(ISD::TRUNCATE, MVT::v2i16, Custom); 859 860 setOperationAction(ISD::AND , MVT::v4i32, Legal); 861 setOperationAction(ISD::OR , MVT::v4i32, Legal); 862 setOperationAction(ISD::XOR , MVT::v4i32, Legal); 863 setOperationAction(ISD::LOAD , MVT::v4i32, Legal); 864 setOperationAction(ISD::SELECT, MVT::v4i32, 865 Subtarget.useCRBits() ? Legal : Expand); 866 setOperationAction(ISD::STORE , MVT::v4i32, Legal); 867 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4i32, Legal); 868 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4i32, Legal); 869 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i32, Legal); 870 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i32, Legal); 871 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal); 872 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal); 873 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal); 874 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal); 875 setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal); 876 setOperationAction(ISD::FCEIL, MVT::v4f32, Legal); 877 setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal); 878 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal); 879 880 // Custom lowering ROTL v1i128 to VECTOR_SHUFFLE v16i8. 881 setOperationAction(ISD::ROTL, MVT::v1i128, Custom); 882 // With hasAltivec set, we can lower ISD::ROTL to vrl(b|h|w). 883 if (Subtarget.hasAltivec()) 884 for (auto VT : {MVT::v4i32, MVT::v8i16, MVT::v16i8}) 885 setOperationAction(ISD::ROTL, VT, Legal); 886 // With hasP8Altivec set, we can lower ISD::ROTL to vrld. 887 if (Subtarget.hasP8Altivec()) 888 setOperationAction(ISD::ROTL, MVT::v2i64, Legal); 889 890 addRegisterClass(MVT::v4f32, &PPC::VRRCRegClass); 891 addRegisterClass(MVT::v4i32, &PPC::VRRCRegClass); 892 addRegisterClass(MVT::v8i16, &PPC::VRRCRegClass); 893 addRegisterClass(MVT::v16i8, &PPC::VRRCRegClass); 894 895 setOperationAction(ISD::MUL, MVT::v4f32, Legal); 896 setOperationAction(ISD::FMA, MVT::v4f32, Legal); 897 898 if (Subtarget.hasVSX()) { 899 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); 900 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); 901 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom); 902 } 903 904 if (Subtarget.hasP8Altivec()) 905 setOperationAction(ISD::MUL, MVT::v4i32, Legal); 906 else 907 setOperationAction(ISD::MUL, MVT::v4i32, Custom); 908 909 if (Subtarget.isISA3_1()) { 910 setOperationAction(ISD::MUL, MVT::v2i64, Legal); 911 setOperationAction(ISD::MULHS, MVT::v2i64, Legal); 912 setOperationAction(ISD::MULHU, MVT::v2i64, Legal); 913 setOperationAction(ISD::MULHS, MVT::v4i32, Legal); 914 setOperationAction(ISD::MULHU, MVT::v4i32, Legal); 915 setOperationAction(ISD::UDIV, MVT::v2i64, Legal); 916 setOperationAction(ISD::SDIV, MVT::v2i64, Legal); 917 setOperationAction(ISD::UDIV, MVT::v4i32, Legal); 918 setOperationAction(ISD::SDIV, MVT::v4i32, Legal); 919 setOperationAction(ISD::UREM, MVT::v2i64, Legal); 920 setOperationAction(ISD::SREM, MVT::v2i64, Legal); 921 setOperationAction(ISD::UREM, MVT::v4i32, Legal); 922 setOperationAction(ISD::SREM, MVT::v4i32, Legal); 923 setOperationAction(ISD::UREM, MVT::v1i128, Legal); 924 setOperationAction(ISD::SREM, MVT::v1i128, Legal); 925 setOperationAction(ISD::UDIV, MVT::v1i128, Legal); 926 setOperationAction(ISD::SDIV, MVT::v1i128, Legal); 927 setOperationAction(ISD::ROTL, MVT::v1i128, Legal); 928 } 929 930 setOperationAction(ISD::MUL, MVT::v8i16, Legal); 931 setOperationAction(ISD::MUL, MVT::v16i8, Custom); 932 933 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Custom); 934 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i32, Custom); 935 936 setOperationAction(ISD::BUILD_VECTOR, MVT::v16i8, Custom); 937 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i16, Custom); 938 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i32, Custom); 939 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); 940 941 // Altivec does not contain unordered floating-point compare instructions 942 setCondCodeAction(ISD::SETUO, MVT::v4f32, Expand); 943 setCondCodeAction(ISD::SETUEQ, MVT::v4f32, Expand); 944 setCondCodeAction(ISD::SETO, MVT::v4f32, Expand); 945 setCondCodeAction(ISD::SETONE, MVT::v4f32, Expand); 946 947 if (Subtarget.hasVSX()) { 948 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal); 949 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Legal); 950 if (Subtarget.hasP8Vector()) { 951 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal); 952 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Legal); 953 } 954 if (Subtarget.hasDirectMove() && isPPC64) { 955 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Legal); 956 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Legal); 957 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i32, Legal); 958 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2i64, Legal); 959 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Legal); 960 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Legal); 961 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Legal); 962 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Legal); 963 } 964 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Legal); 965 966 // The nearbyint variants are not allowed to raise the inexact exception 967 // so we can only code-gen them with unsafe math. 968 if (TM.Options.UnsafeFPMath) { 969 setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal); 970 setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal); 971 } 972 973 setOperationAction(ISD::FFLOOR, MVT::v2f64, Legal); 974 setOperationAction(ISD::FCEIL, MVT::v2f64, Legal); 975 setOperationAction(ISD::FTRUNC, MVT::v2f64, Legal); 976 setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Legal); 977 setOperationAction(ISD::FRINT, MVT::v2f64, Legal); 978 setOperationAction(ISD::FROUND, MVT::v2f64, Legal); 979 setOperationAction(ISD::FROUND, MVT::f64, Legal); 980 setOperationAction(ISD::FRINT, MVT::f64, Legal); 981 982 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal); 983 setOperationAction(ISD::FRINT, MVT::v4f32, Legal); 984 setOperationAction(ISD::FROUND, MVT::v4f32, Legal); 985 setOperationAction(ISD::FROUND, MVT::f32, Legal); 986 setOperationAction(ISD::FRINT, MVT::f32, Legal); 987 988 setOperationAction(ISD::MUL, MVT::v2f64, Legal); 989 setOperationAction(ISD::FMA, MVT::v2f64, Legal); 990 991 setOperationAction(ISD::FDIV, MVT::v2f64, Legal); 992 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); 993 994 // Share the Altivec comparison restrictions. 995 setCondCodeAction(ISD::SETUO, MVT::v2f64, Expand); 996 setCondCodeAction(ISD::SETUEQ, MVT::v2f64, Expand); 997 setCondCodeAction(ISD::SETO, MVT::v2f64, Expand); 998 setCondCodeAction(ISD::SETONE, MVT::v2f64, Expand); 999 1000 setOperationAction(ISD::LOAD, MVT::v2f64, Legal); 1001 setOperationAction(ISD::STORE, MVT::v2f64, Legal); 1002 1003 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Legal); 1004 1005 if (Subtarget.hasP8Vector()) 1006 addRegisterClass(MVT::f32, &PPC::VSSRCRegClass); 1007 1008 addRegisterClass(MVT::f64, &PPC::VSFRCRegClass); 1009 1010 addRegisterClass(MVT::v4i32, &PPC::VSRCRegClass); 1011 addRegisterClass(MVT::v4f32, &PPC::VSRCRegClass); 1012 addRegisterClass(MVT::v2f64, &PPC::VSRCRegClass); 1013 1014 if (Subtarget.hasP8Altivec()) { 1015 setOperationAction(ISD::SHL, MVT::v2i64, Legal); 1016 setOperationAction(ISD::SRA, MVT::v2i64, Legal); 1017 setOperationAction(ISD::SRL, MVT::v2i64, Legal); 1018 1019 // 128 bit shifts can be accomplished via 3 instructions for SHL and 1020 // SRL, but not for SRA because of the instructions available: 1021 // VS{RL} and VS{RL}O. However due to direct move costs, it's not worth 1022 // doing 1023 setOperationAction(ISD::SHL, MVT::v1i128, Expand); 1024 setOperationAction(ISD::SRL, MVT::v1i128, Expand); 1025 setOperationAction(ISD::SRA, MVT::v1i128, Expand); 1026 1027 setOperationAction(ISD::SETCC, MVT::v2i64, Legal); 1028 } 1029 else { 1030 setOperationAction(ISD::SHL, MVT::v2i64, Expand); 1031 setOperationAction(ISD::SRA, MVT::v2i64, Expand); 1032 setOperationAction(ISD::SRL, MVT::v2i64, Expand); 1033 1034 setOperationAction(ISD::SETCC, MVT::v2i64, Custom); 1035 1036 // VSX v2i64 only supports non-arithmetic operations. 1037 setOperationAction(ISD::ADD, MVT::v2i64, Expand); 1038 setOperationAction(ISD::SUB, MVT::v2i64, Expand); 1039 } 1040 1041 if (Subtarget.isISA3_1()) 1042 setOperationAction(ISD::SETCC, MVT::v1i128, Legal); 1043 else 1044 setOperationAction(ISD::SETCC, MVT::v1i128, Expand); 1045 1046 setOperationAction(ISD::LOAD, MVT::v2i64, Promote); 1047 AddPromotedToType (ISD::LOAD, MVT::v2i64, MVT::v2f64); 1048 setOperationAction(ISD::STORE, MVT::v2i64, Promote); 1049 AddPromotedToType (ISD::STORE, MVT::v2i64, MVT::v2f64); 1050 1051 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Legal); 1052 1053 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i64, Legal); 1054 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i64, Legal); 1055 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i64, Legal); 1056 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i64, Legal); 1057 setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Legal); 1058 setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Legal); 1059 setOperationAction(ISD::FP_TO_SINT, MVT::v2i64, Legal); 1060 setOperationAction(ISD::FP_TO_UINT, MVT::v2i64, Legal); 1061 1062 // Custom handling for partial vectors of integers converted to 1063 // floating point. We already have optimal handling for v2i32 through 1064 // the DAG combine, so those aren't necessary. 1065 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i8, Custom); 1066 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i8, Custom); 1067 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i16, Custom); 1068 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i16, Custom); 1069 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i8, Custom); 1070 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i8, Custom); 1071 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i16, Custom); 1072 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i16, Custom); 1073 setOperationAction(ISD::UINT_TO_FP, MVT::v2i8, Custom); 1074 setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Custom); 1075 setOperationAction(ISD::UINT_TO_FP, MVT::v2i16, Custom); 1076 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom); 1077 setOperationAction(ISD::SINT_TO_FP, MVT::v2i8, Custom); 1078 setOperationAction(ISD::SINT_TO_FP, MVT::v4i8, Custom); 1079 setOperationAction(ISD::SINT_TO_FP, MVT::v2i16, Custom); 1080 setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom); 1081 1082 setOperationAction(ISD::FNEG, MVT::v4f32, Legal); 1083 setOperationAction(ISD::FNEG, MVT::v2f64, Legal); 1084 setOperationAction(ISD::FABS, MVT::v4f32, Legal); 1085 setOperationAction(ISD::FABS, MVT::v2f64, Legal); 1086 setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Legal); 1087 setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Legal); 1088 1089 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); 1090 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom); 1091 1092 // Handle constrained floating-point operations of vector. 1093 // The predictor is `hasVSX` because altivec instruction has 1094 // no exception but VSX vector instruction has. 1095 setOperationAction(ISD::STRICT_FADD, MVT::v4f32, Legal); 1096 setOperationAction(ISD::STRICT_FSUB, MVT::v4f32, Legal); 1097 setOperationAction(ISD::STRICT_FMUL, MVT::v4f32, Legal); 1098 setOperationAction(ISD::STRICT_FDIV, MVT::v4f32, Legal); 1099 setOperationAction(ISD::STRICT_FMA, MVT::v4f32, Legal); 1100 setOperationAction(ISD::STRICT_FSQRT, MVT::v4f32, Legal); 1101 setOperationAction(ISD::STRICT_FMAXNUM, MVT::v4f32, Legal); 1102 setOperationAction(ISD::STRICT_FMINNUM, MVT::v4f32, Legal); 1103 setOperationAction(ISD::STRICT_FRINT, MVT::v4f32, Legal); 1104 setOperationAction(ISD::STRICT_FFLOOR, MVT::v4f32, Legal); 1105 setOperationAction(ISD::STRICT_FCEIL, MVT::v4f32, Legal); 1106 setOperationAction(ISD::STRICT_FTRUNC, MVT::v4f32, Legal); 1107 setOperationAction(ISD::STRICT_FROUND, MVT::v4f32, Legal); 1108 1109 setOperationAction(ISD::STRICT_FADD, MVT::v2f64, Legal); 1110 setOperationAction(ISD::STRICT_FSUB, MVT::v2f64, Legal); 1111 setOperationAction(ISD::STRICT_FMUL, MVT::v2f64, Legal); 1112 setOperationAction(ISD::STRICT_FDIV, MVT::v2f64, Legal); 1113 setOperationAction(ISD::STRICT_FMA, MVT::v2f64, Legal); 1114 setOperationAction(ISD::STRICT_FSQRT, MVT::v2f64, Legal); 1115 setOperationAction(ISD::STRICT_FMAXNUM, MVT::v2f64, Legal); 1116 setOperationAction(ISD::STRICT_FMINNUM, MVT::v2f64, Legal); 1117 setOperationAction(ISD::STRICT_FRINT, MVT::v2f64, Legal); 1118 setOperationAction(ISD::STRICT_FFLOOR, MVT::v2f64, Legal); 1119 setOperationAction(ISD::STRICT_FCEIL, MVT::v2f64, Legal); 1120 setOperationAction(ISD::STRICT_FTRUNC, MVT::v2f64, Legal); 1121 setOperationAction(ISD::STRICT_FROUND, MVT::v2f64, Legal); 1122 1123 addRegisterClass(MVT::v2i64, &PPC::VSRCRegClass); 1124 addRegisterClass(MVT::f128, &PPC::VRRCRegClass); 1125 1126 for (MVT FPT : MVT::fp_valuetypes()) 1127 setLoadExtAction(ISD::EXTLOAD, MVT::f128, FPT, Expand); 1128 1129 // Expand the SELECT to SELECT_CC 1130 setOperationAction(ISD::SELECT, MVT::f128, Expand); 1131 1132 setTruncStoreAction(MVT::f128, MVT::f64, Expand); 1133 setTruncStoreAction(MVT::f128, MVT::f32, Expand); 1134 1135 // No implementation for these ops for PowerPC. 1136 setOperationAction(ISD::FSIN, MVT::f128, Expand); 1137 setOperationAction(ISD::FCOS, MVT::f128, Expand); 1138 setOperationAction(ISD::FPOW, MVT::f128, Expand); 1139 setOperationAction(ISD::FPOWI, MVT::f128, Expand); 1140 setOperationAction(ISD::FREM, MVT::f128, Expand); 1141 } 1142 1143 if (Subtarget.hasP8Altivec()) { 1144 addRegisterClass(MVT::v2i64, &PPC::VRRCRegClass); 1145 addRegisterClass(MVT::v1i128, &PPC::VRRCRegClass); 1146 } 1147 1148 if (Subtarget.hasP9Vector()) { 1149 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); 1150 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 1151 1152 // 128 bit shifts can be accomplished via 3 instructions for SHL and 1153 // SRL, but not for SRA because of the instructions available: 1154 // VS{RL} and VS{RL}O. 1155 setOperationAction(ISD::SHL, MVT::v1i128, Legal); 1156 setOperationAction(ISD::SRL, MVT::v1i128, Legal); 1157 setOperationAction(ISD::SRA, MVT::v1i128, Expand); 1158 1159 setOperationAction(ISD::FADD, MVT::f128, Legal); 1160 setOperationAction(ISD::FSUB, MVT::f128, Legal); 1161 setOperationAction(ISD::FDIV, MVT::f128, Legal); 1162 setOperationAction(ISD::FMUL, MVT::f128, Legal); 1163 setOperationAction(ISD::FP_EXTEND, MVT::f128, Legal); 1164 1165 setOperationAction(ISD::FMA, MVT::f128, Legal); 1166 setCondCodeAction(ISD::SETULT, MVT::f128, Expand); 1167 setCondCodeAction(ISD::SETUGT, MVT::f128, Expand); 1168 setCondCodeAction(ISD::SETUEQ, MVT::f128, Expand); 1169 setCondCodeAction(ISD::SETOGE, MVT::f128, Expand); 1170 setCondCodeAction(ISD::SETOLE, MVT::f128, Expand); 1171 setCondCodeAction(ISD::SETONE, MVT::f128, Expand); 1172 1173 setOperationAction(ISD::FTRUNC, MVT::f128, Legal); 1174 setOperationAction(ISD::FRINT, MVT::f128, Legal); 1175 setOperationAction(ISD::FFLOOR, MVT::f128, Legal); 1176 setOperationAction(ISD::FCEIL, MVT::f128, Legal); 1177 setOperationAction(ISD::FNEARBYINT, MVT::f128, Legal); 1178 setOperationAction(ISD::FROUND, MVT::f128, Legal); 1179 1180 setOperationAction(ISD::FP_ROUND, MVT::f64, Legal); 1181 setOperationAction(ISD::FP_ROUND, MVT::f32, Legal); 1182 setOperationAction(ISD::BITCAST, MVT::i128, Custom); 1183 1184 // Handle constrained floating-point operations of fp128 1185 setOperationAction(ISD::STRICT_FADD, MVT::f128, Legal); 1186 setOperationAction(ISD::STRICT_FSUB, MVT::f128, Legal); 1187 setOperationAction(ISD::STRICT_FMUL, MVT::f128, Legal); 1188 setOperationAction(ISD::STRICT_FDIV, MVT::f128, Legal); 1189 setOperationAction(ISD::STRICT_FMA, MVT::f128, Legal); 1190 setOperationAction(ISD::STRICT_FSQRT, MVT::f128, Legal); 1191 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f128, Legal); 1192 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Legal); 1193 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal); 1194 setOperationAction(ISD::STRICT_FRINT, MVT::f128, Legal); 1195 setOperationAction(ISD::STRICT_FNEARBYINT, MVT::f128, Legal); 1196 setOperationAction(ISD::STRICT_FFLOOR, MVT::f128, Legal); 1197 setOperationAction(ISD::STRICT_FCEIL, MVT::f128, Legal); 1198 setOperationAction(ISD::STRICT_FTRUNC, MVT::f128, Legal); 1199 setOperationAction(ISD::STRICT_FROUND, MVT::f128, Legal); 1200 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom); 1201 setOperationAction(ISD::BSWAP, MVT::v8i16, Legal); 1202 setOperationAction(ISD::BSWAP, MVT::v4i32, Legal); 1203 setOperationAction(ISD::BSWAP, MVT::v2i64, Legal); 1204 setOperationAction(ISD::BSWAP, MVT::v1i128, Legal); 1205 } else if (Subtarget.hasVSX()) { 1206 setOperationAction(ISD::LOAD, MVT::f128, Promote); 1207 setOperationAction(ISD::STORE, MVT::f128, Promote); 1208 1209 AddPromotedToType(ISD::LOAD, MVT::f128, MVT::v4i32); 1210 AddPromotedToType(ISD::STORE, MVT::f128, MVT::v4i32); 1211 1212 // Set FADD/FSUB as libcall to avoid the legalizer to expand the 1213 // fp_to_uint and int_to_fp. 1214 setOperationAction(ISD::FADD, MVT::f128, LibCall); 1215 setOperationAction(ISD::FSUB, MVT::f128, LibCall); 1216 1217 setOperationAction(ISD::FMUL, MVT::f128, Expand); 1218 setOperationAction(ISD::FDIV, MVT::f128, Expand); 1219 setOperationAction(ISD::FNEG, MVT::f128, Expand); 1220 setOperationAction(ISD::FABS, MVT::f128, Expand); 1221 setOperationAction(ISD::FSQRT, MVT::f128, Expand); 1222 setOperationAction(ISD::FMA, MVT::f128, Expand); 1223 setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand); 1224 1225 // Expand the fp_extend if the target type is fp128. 1226 setOperationAction(ISD::FP_EXTEND, MVT::f128, Expand); 1227 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f128, Expand); 1228 1229 // Expand the fp_round if the source type is fp128. 1230 for (MVT VT : {MVT::f32, MVT::f64}) { 1231 setOperationAction(ISD::FP_ROUND, VT, Custom); 1232 setOperationAction(ISD::STRICT_FP_ROUND, VT, Custom); 1233 } 1234 1235 setOperationAction(ISD::SETCC, MVT::f128, Custom); 1236 setOperationAction(ISD::STRICT_FSETCC, MVT::f128, Custom); 1237 setOperationAction(ISD::STRICT_FSETCCS, MVT::f128, Custom); 1238 setOperationAction(ISD::BR_CC, MVT::f128, Expand); 1239 1240 // Lower following f128 select_cc pattern: 1241 // select_cc x, y, tv, fv, cc -> select_cc (setcc x, y, cc), 0, tv, fv, NE 1242 setOperationAction(ISD::SELECT_CC, MVT::f128, Custom); 1243 1244 // We need to handle f128 SELECT_CC with integer result type. 1245 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom); 1246 setOperationAction(ISD::SELECT_CC, MVT::i64, isPPC64 ? Custom : Expand); 1247 } 1248 1249 if (Subtarget.hasP9Altivec()) { 1250 if (Subtarget.isISA3_1()) { 1251 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Legal); 1252 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Legal); 1253 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Legal); 1254 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Legal); 1255 } else { 1256 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 1257 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom); 1258 } 1259 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i8, Legal); 1260 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i16, Legal); 1261 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i32, Legal); 1262 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Legal); 1263 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Legal); 1264 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Legal); 1265 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i64, Legal); 1266 } 1267 } 1268 1269 if (Subtarget.pairedVectorMemops()) { 1270 addRegisterClass(MVT::v256i1, &PPC::VSRpRCRegClass); 1271 setOperationAction(ISD::LOAD, MVT::v256i1, Custom); 1272 setOperationAction(ISD::STORE, MVT::v256i1, Custom); 1273 } 1274 if (Subtarget.hasMMA()) { 1275 addRegisterClass(MVT::v512i1, &PPC::UACCRCRegClass); 1276 setOperationAction(ISD::LOAD, MVT::v512i1, Custom); 1277 setOperationAction(ISD::STORE, MVT::v512i1, Custom); 1278 setOperationAction(ISD::BUILD_VECTOR, MVT::v512i1, Custom); 1279 } 1280 1281 if (Subtarget.has64BitSupport()) 1282 setOperationAction(ISD::PREFETCH, MVT::Other, Legal); 1283 1284 if (Subtarget.isISA3_1()) 1285 setOperationAction(ISD::SRA, MVT::v1i128, Legal); 1286 1287 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, isPPC64 ? Legal : Custom); 1288 1289 if (!isPPC64) { 1290 setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Expand); 1291 setOperationAction(ISD::ATOMIC_STORE, MVT::i64, Expand); 1292 } 1293 1294 if (EnableQuadwordAtomics && Subtarget.hasQuadwordAtomics()) { 1295 setMaxAtomicSizeInBitsSupported(128); 1296 setOperationAction(ISD::ATOMIC_LOAD, MVT::i128, Custom); 1297 setOperationAction(ISD::ATOMIC_STORE, MVT::i128, Custom); 1298 setOperationAction(ISD::INTRINSIC_VOID, MVT::i128, Custom); 1299 } 1300 1301 setBooleanContents(ZeroOrOneBooleanContent); 1302 1303 if (Subtarget.hasAltivec()) { 1304 // Altivec instructions set fields to all zeros or all ones. 1305 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); 1306 } 1307 1308 setLibcallName(RTLIB::MULO_I128, nullptr); 1309 if (!isPPC64) { 1310 // These libcalls are not available in 32-bit. 1311 setLibcallName(RTLIB::SHL_I128, nullptr); 1312 setLibcallName(RTLIB::SRL_I128, nullptr); 1313 setLibcallName(RTLIB::SRA_I128, nullptr); 1314 setLibcallName(RTLIB::MUL_I128, nullptr); 1315 setLibcallName(RTLIB::MULO_I64, nullptr); 1316 setLibcallName(RTLIB::MULO_I128, nullptr); 1317 } 1318 1319 if (!isPPC64) 1320 setMaxAtomicSizeInBitsSupported(32); 1321 1322 setStackPointerRegisterToSaveRestore(isPPC64 ? PPC::X1 : PPC::R1); 1323 1324 // We have target-specific dag combine patterns for the following nodes: 1325 setTargetDAGCombine(ISD::ADD); 1326 setTargetDAGCombine(ISD::SHL); 1327 setTargetDAGCombine(ISD::SRA); 1328 setTargetDAGCombine(ISD::SRL); 1329 setTargetDAGCombine(ISD::MUL); 1330 setTargetDAGCombine(ISD::FMA); 1331 setTargetDAGCombine(ISD::SINT_TO_FP); 1332 setTargetDAGCombine(ISD::BUILD_VECTOR); 1333 if (Subtarget.hasFPCVT()) 1334 setTargetDAGCombine(ISD::UINT_TO_FP); 1335 setTargetDAGCombine(ISD::LOAD); 1336 setTargetDAGCombine(ISD::STORE); 1337 setTargetDAGCombine(ISD::BR_CC); 1338 if (Subtarget.useCRBits()) 1339 setTargetDAGCombine(ISD::BRCOND); 1340 setTargetDAGCombine(ISD::BSWAP); 1341 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN); 1342 setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN); 1343 setTargetDAGCombine(ISD::INTRINSIC_VOID); 1344 1345 setTargetDAGCombine(ISD::SIGN_EXTEND); 1346 setTargetDAGCombine(ISD::ZERO_EXTEND); 1347 setTargetDAGCombine(ISD::ANY_EXTEND); 1348 1349 setTargetDAGCombine(ISD::TRUNCATE); 1350 setTargetDAGCombine(ISD::VECTOR_SHUFFLE); 1351 1352 1353 if (Subtarget.useCRBits()) { 1354 setTargetDAGCombine(ISD::TRUNCATE); 1355 setTargetDAGCombine(ISD::SETCC); 1356 setTargetDAGCombine(ISD::SELECT_CC); 1357 } 1358 1359 if (Subtarget.hasP9Altivec()) { 1360 setTargetDAGCombine(ISD::ABS); 1361 setTargetDAGCombine(ISD::VSELECT); 1362 } 1363 1364 setLibcallName(RTLIB::LOG_F128, "logf128"); 1365 setLibcallName(RTLIB::LOG2_F128, "log2f128"); 1366 setLibcallName(RTLIB::LOG10_F128, "log10f128"); 1367 setLibcallName(RTLIB::EXP_F128, "expf128"); 1368 setLibcallName(RTLIB::EXP2_F128, "exp2f128"); 1369 setLibcallName(RTLIB::SIN_F128, "sinf128"); 1370 setLibcallName(RTLIB::COS_F128, "cosf128"); 1371 setLibcallName(RTLIB::POW_F128, "powf128"); 1372 setLibcallName(RTLIB::FMIN_F128, "fminf128"); 1373 setLibcallName(RTLIB::FMAX_F128, "fmaxf128"); 1374 setLibcallName(RTLIB::REM_F128, "fmodf128"); 1375 setLibcallName(RTLIB::SQRT_F128, "sqrtf128"); 1376 setLibcallName(RTLIB::CEIL_F128, "ceilf128"); 1377 setLibcallName(RTLIB::FLOOR_F128, "floorf128"); 1378 setLibcallName(RTLIB::TRUNC_F128, "truncf128"); 1379 setLibcallName(RTLIB::ROUND_F128, "roundf128"); 1380 setLibcallName(RTLIB::LROUND_F128, "lroundf128"); 1381 setLibcallName(RTLIB::LLROUND_F128, "llroundf128"); 1382 setLibcallName(RTLIB::RINT_F128, "rintf128"); 1383 setLibcallName(RTLIB::LRINT_F128, "lrintf128"); 1384 setLibcallName(RTLIB::LLRINT_F128, "llrintf128"); 1385 setLibcallName(RTLIB::NEARBYINT_F128, "nearbyintf128"); 1386 setLibcallName(RTLIB::FMA_F128, "fmaf128"); 1387 1388 // With 32 condition bits, we don't need to sink (and duplicate) compares 1389 // aggressively in CodeGenPrep. 1390 if (Subtarget.useCRBits()) { 1391 setHasMultipleConditionRegisters(); 1392 setJumpIsExpensive(); 1393 } 1394 1395 setMinFunctionAlignment(Align(4)); 1396 1397 switch (Subtarget.getCPUDirective()) { 1398 default: break; 1399 case PPC::DIR_970: 1400 case PPC::DIR_A2: 1401 case PPC::DIR_E500: 1402 case PPC::DIR_E500mc: 1403 case PPC::DIR_E5500: 1404 case PPC::DIR_PWR4: 1405 case PPC::DIR_PWR5: 1406 case PPC::DIR_PWR5X: 1407 case PPC::DIR_PWR6: 1408 case PPC::DIR_PWR6X: 1409 case PPC::DIR_PWR7: 1410 case PPC::DIR_PWR8: 1411 case PPC::DIR_PWR9: 1412 case PPC::DIR_PWR10: 1413 case PPC::DIR_PWR_FUTURE: 1414 setPrefLoopAlignment(Align(16)); 1415 setPrefFunctionAlignment(Align(16)); 1416 break; 1417 } 1418 1419 if (Subtarget.enableMachineScheduler()) 1420 setSchedulingPreference(Sched::Source); 1421 else 1422 setSchedulingPreference(Sched::Hybrid); 1423 1424 computeRegisterProperties(STI.getRegisterInfo()); 1425 1426 // The Freescale cores do better with aggressive inlining of memcpy and 1427 // friends. GCC uses same threshold of 128 bytes (= 32 word stores). 1428 if (Subtarget.getCPUDirective() == PPC::DIR_E500mc || 1429 Subtarget.getCPUDirective() == PPC::DIR_E5500) { 1430 MaxStoresPerMemset = 32; 1431 MaxStoresPerMemsetOptSize = 16; 1432 MaxStoresPerMemcpy = 32; 1433 MaxStoresPerMemcpyOptSize = 8; 1434 MaxStoresPerMemmove = 32; 1435 MaxStoresPerMemmoveOptSize = 8; 1436 } else if (Subtarget.getCPUDirective() == PPC::DIR_A2) { 1437 // The A2 also benefits from (very) aggressive inlining of memcpy and 1438 // friends. The overhead of a the function call, even when warm, can be 1439 // over one hundred cycles. 1440 MaxStoresPerMemset = 128; 1441 MaxStoresPerMemcpy = 128; 1442 MaxStoresPerMemmove = 128; 1443 MaxLoadsPerMemcmp = 128; 1444 } else { 1445 MaxLoadsPerMemcmp = 8; 1446 MaxLoadsPerMemcmpOptSize = 4; 1447 } 1448 1449 IsStrictFPEnabled = true; 1450 1451 // Let the subtarget (CPU) decide if a predictable select is more expensive 1452 // than the corresponding branch. This information is used in CGP to decide 1453 // when to convert selects into branches. 1454 PredictableSelectIsExpensive = Subtarget.isPredictableSelectIsExpensive(); 1455 } 1456 1457 // *********************************** NOTE ************************************ 1458 // For selecting load and store instructions, the addressing modes are defined 1459 // as ComplexPatterns in PPCInstrInfo.td, which are then utilized in the TD 1460 // patterns to match the load the store instructions. 1461 // 1462 // The TD definitions for the addressing modes correspond to their respective 1463 // Select<AddrMode>Form() function in PPCISelDAGToDAG.cpp. These functions rely 1464 // on SelectOptimalAddrMode(), which calls computeMOFlags() to compute the 1465 // address mode flags of a particular node. Afterwards, the computed address 1466 // flags are passed into getAddrModeForFlags() in order to retrieve the optimal 1467 // addressing mode. SelectOptimalAddrMode() then sets the Base and Displacement 1468 // accordingly, based on the preferred addressing mode. 1469 // 1470 // Within PPCISelLowering.h, there are two enums: MemOpFlags and AddrMode. 1471 // MemOpFlags contains all the possible flags that can be used to compute the 1472 // optimal addressing mode for load and store instructions. 1473 // AddrMode contains all the possible load and store addressing modes available 1474 // on Power (such as DForm, DSForm, DQForm, XForm, etc.) 1475 // 1476 // When adding new load and store instructions, it is possible that new address 1477 // flags may need to be added into MemOpFlags, and a new addressing mode will 1478 // need to be added to AddrMode. An entry of the new addressing mode (consisting 1479 // of the minimal and main distinguishing address flags for the new load/store 1480 // instructions) will need to be added into initializeAddrModeMap() below. 1481 // Finally, when adding new addressing modes, the getAddrModeForFlags() will 1482 // need to be updated to account for selecting the optimal addressing mode. 1483 // ***************************************************************************** 1484 /// Initialize the map that relates the different addressing modes of the load 1485 /// and store instructions to a set of flags. This ensures the load/store 1486 /// instruction is correctly matched during instruction selection. 1487 void PPCTargetLowering::initializeAddrModeMap() { 1488 AddrModesMap[PPC::AM_DForm] = { 1489 // LWZ, STW 1490 PPC::MOF_ZExt | PPC::MOF_RPlusSImm16 | PPC::MOF_WordInt, 1491 PPC::MOF_ZExt | PPC::MOF_RPlusLo | PPC::MOF_WordInt, 1492 PPC::MOF_ZExt | PPC::MOF_NotAddNorCst | PPC::MOF_WordInt, 1493 PPC::MOF_ZExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_WordInt, 1494 // LBZ, LHZ, STB, STH 1495 PPC::MOF_ZExt | PPC::MOF_RPlusSImm16 | PPC::MOF_SubWordInt, 1496 PPC::MOF_ZExt | PPC::MOF_RPlusLo | PPC::MOF_SubWordInt, 1497 PPC::MOF_ZExt | PPC::MOF_NotAddNorCst | PPC::MOF_SubWordInt, 1498 PPC::MOF_ZExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubWordInt, 1499 // LHA 1500 PPC::MOF_SExt | PPC::MOF_RPlusSImm16 | PPC::MOF_SubWordInt, 1501 PPC::MOF_SExt | PPC::MOF_RPlusLo | PPC::MOF_SubWordInt, 1502 PPC::MOF_SExt | PPC::MOF_NotAddNorCst | PPC::MOF_SubWordInt, 1503 PPC::MOF_SExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubWordInt, 1504 // LFS, LFD, STFS, STFD 1505 PPC::MOF_RPlusSImm16 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9, 1506 PPC::MOF_RPlusLo | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9, 1507 PPC::MOF_NotAddNorCst | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9, 1508 PPC::MOF_AddrIsSImm32 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9, 1509 }; 1510 AddrModesMap[PPC::AM_DSForm] = { 1511 // LWA 1512 PPC::MOF_SExt | PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_WordInt, 1513 PPC::MOF_SExt | PPC::MOF_NotAddNorCst | PPC::MOF_WordInt, 1514 PPC::MOF_SExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_WordInt, 1515 // LD, STD 1516 PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_DoubleWordInt, 1517 PPC::MOF_NotAddNorCst | PPC::MOF_DoubleWordInt, 1518 PPC::MOF_AddrIsSImm32 | PPC::MOF_DoubleWordInt, 1519 // DFLOADf32, DFLOADf64, DSTOREf32, DSTOREf64 1520 PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9, 1521 PPC::MOF_NotAddNorCst | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9, 1522 PPC::MOF_AddrIsSImm32 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9, 1523 }; 1524 AddrModesMap[PPC::AM_DQForm] = { 1525 // LXV, STXV 1526 PPC::MOF_RPlusSImm16Mult16 | PPC::MOF_Vector | PPC::MOF_SubtargetP9, 1527 PPC::MOF_NotAddNorCst | PPC::MOF_Vector | PPC::MOF_SubtargetP9, 1528 PPC::MOF_AddrIsSImm32 | PPC::MOF_Vector | PPC::MOF_SubtargetP9, 1529 }; 1530 AddrModesMap[PPC::AM_PrefixDForm] = {PPC::MOF_RPlusSImm34 | 1531 PPC::MOF_SubtargetP10}; 1532 // TODO: Add mapping for quadword load/store. 1533 } 1534 1535 /// getMaxByValAlign - Helper for getByValTypeAlignment to determine 1536 /// the desired ByVal argument alignment. 1537 static void getMaxByValAlign(Type *Ty, Align &MaxAlign, Align MaxMaxAlign) { 1538 if (MaxAlign == MaxMaxAlign) 1539 return; 1540 if (VectorType *VTy = dyn_cast<VectorType>(Ty)) { 1541 if (MaxMaxAlign >= 32 && 1542 VTy->getPrimitiveSizeInBits().getFixedSize() >= 256) 1543 MaxAlign = Align(32); 1544 else if (VTy->getPrimitiveSizeInBits().getFixedSize() >= 128 && 1545 MaxAlign < 16) 1546 MaxAlign = Align(16); 1547 } else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { 1548 Align EltAlign; 1549 getMaxByValAlign(ATy->getElementType(), EltAlign, MaxMaxAlign); 1550 if (EltAlign > MaxAlign) 1551 MaxAlign = EltAlign; 1552 } else if (StructType *STy = dyn_cast<StructType>(Ty)) { 1553 for (auto *EltTy : STy->elements()) { 1554 Align EltAlign; 1555 getMaxByValAlign(EltTy, EltAlign, MaxMaxAlign); 1556 if (EltAlign > MaxAlign) 1557 MaxAlign = EltAlign; 1558 if (MaxAlign == MaxMaxAlign) 1559 break; 1560 } 1561 } 1562 } 1563 1564 /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate 1565 /// function arguments in the caller parameter area. 1566 uint64_t PPCTargetLowering::getByValTypeAlignment(Type *Ty, 1567 const DataLayout &DL) const { 1568 // 16byte and wider vectors are passed on 16byte boundary. 1569 // The rest is 8 on PPC64 and 4 on PPC32 boundary. 1570 Align Alignment = Subtarget.isPPC64() ? Align(8) : Align(4); 1571 if (Subtarget.hasAltivec()) 1572 getMaxByValAlign(Ty, Alignment, Align(16)); 1573 return Alignment.value(); 1574 } 1575 1576 bool PPCTargetLowering::useSoftFloat() const { 1577 return Subtarget.useSoftFloat(); 1578 } 1579 1580 bool PPCTargetLowering::hasSPE() const { 1581 return Subtarget.hasSPE(); 1582 } 1583 1584 bool PPCTargetLowering::preferIncOfAddToSubOfNot(EVT VT) const { 1585 return VT.isScalarInteger(); 1586 } 1587 1588 const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const { 1589 switch ((PPCISD::NodeType)Opcode) { 1590 case PPCISD::FIRST_NUMBER: break; 1591 case PPCISD::FSEL: return "PPCISD::FSEL"; 1592 case PPCISD::XSMAXCDP: return "PPCISD::XSMAXCDP"; 1593 case PPCISD::XSMINCDP: return "PPCISD::XSMINCDP"; 1594 case PPCISD::FCFID: return "PPCISD::FCFID"; 1595 case PPCISD::FCFIDU: return "PPCISD::FCFIDU"; 1596 case PPCISD::FCFIDS: return "PPCISD::FCFIDS"; 1597 case PPCISD::FCFIDUS: return "PPCISD::FCFIDUS"; 1598 case PPCISD::FCTIDZ: return "PPCISD::FCTIDZ"; 1599 case PPCISD::FCTIWZ: return "PPCISD::FCTIWZ"; 1600 case PPCISD::FCTIDUZ: return "PPCISD::FCTIDUZ"; 1601 case PPCISD::FCTIWUZ: return "PPCISD::FCTIWUZ"; 1602 case PPCISD::FP_TO_UINT_IN_VSR: 1603 return "PPCISD::FP_TO_UINT_IN_VSR,"; 1604 case PPCISD::FP_TO_SINT_IN_VSR: 1605 return "PPCISD::FP_TO_SINT_IN_VSR"; 1606 case PPCISD::FRE: return "PPCISD::FRE"; 1607 case PPCISD::FRSQRTE: return "PPCISD::FRSQRTE"; 1608 case PPCISD::FTSQRT: 1609 return "PPCISD::FTSQRT"; 1610 case PPCISD::FSQRT: 1611 return "PPCISD::FSQRT"; 1612 case PPCISD::STFIWX: return "PPCISD::STFIWX"; 1613 case PPCISD::VPERM: return "PPCISD::VPERM"; 1614 case PPCISD::XXSPLT: return "PPCISD::XXSPLT"; 1615 case PPCISD::XXSPLTI_SP_TO_DP: 1616 return "PPCISD::XXSPLTI_SP_TO_DP"; 1617 case PPCISD::XXSPLTI32DX: 1618 return "PPCISD::XXSPLTI32DX"; 1619 case PPCISD::VECINSERT: return "PPCISD::VECINSERT"; 1620 case PPCISD::XXPERMDI: return "PPCISD::XXPERMDI"; 1621 case PPCISD::VECSHL: return "PPCISD::VECSHL"; 1622 case PPCISD::CMPB: return "PPCISD::CMPB"; 1623 case PPCISD::Hi: return "PPCISD::Hi"; 1624 case PPCISD::Lo: return "PPCISD::Lo"; 1625 case PPCISD::TOC_ENTRY: return "PPCISD::TOC_ENTRY"; 1626 case PPCISD::ATOMIC_CMP_SWAP_8: return "PPCISD::ATOMIC_CMP_SWAP_8"; 1627 case PPCISD::ATOMIC_CMP_SWAP_16: return "PPCISD::ATOMIC_CMP_SWAP_16"; 1628 case PPCISD::DYNALLOC: return "PPCISD::DYNALLOC"; 1629 case PPCISD::DYNAREAOFFSET: return "PPCISD::DYNAREAOFFSET"; 1630 case PPCISD::PROBED_ALLOCA: return "PPCISD::PROBED_ALLOCA"; 1631 case PPCISD::GlobalBaseReg: return "PPCISD::GlobalBaseReg"; 1632 case PPCISD::SRL: return "PPCISD::SRL"; 1633 case PPCISD::SRA: return "PPCISD::SRA"; 1634 case PPCISD::SHL: return "PPCISD::SHL"; 1635 case PPCISD::SRA_ADDZE: return "PPCISD::SRA_ADDZE"; 1636 case PPCISD::CALL: return "PPCISD::CALL"; 1637 case PPCISD::CALL_NOP: return "PPCISD::CALL_NOP"; 1638 case PPCISD::CALL_NOTOC: return "PPCISD::CALL_NOTOC"; 1639 case PPCISD::CALL_RM: 1640 return "PPCISD::CALL_RM"; 1641 case PPCISD::CALL_NOP_RM: 1642 return "PPCISD::CALL_NOP_RM"; 1643 case PPCISD::CALL_NOTOC_RM: 1644 return "PPCISD::CALL_NOTOC_RM"; 1645 case PPCISD::MTCTR: return "PPCISD::MTCTR"; 1646 case PPCISD::BCTRL: return "PPCISD::BCTRL"; 1647 case PPCISD::BCTRL_LOAD_TOC: return "PPCISD::BCTRL_LOAD_TOC"; 1648 case PPCISD::BCTRL_RM: 1649 return "PPCISD::BCTRL_RM"; 1650 case PPCISD::BCTRL_LOAD_TOC_RM: 1651 return "PPCISD::BCTRL_LOAD_TOC_RM"; 1652 case PPCISD::RET_FLAG: return "PPCISD::RET_FLAG"; 1653 case PPCISD::READ_TIME_BASE: return "PPCISD::READ_TIME_BASE"; 1654 case PPCISD::EH_SJLJ_SETJMP: return "PPCISD::EH_SJLJ_SETJMP"; 1655 case PPCISD::EH_SJLJ_LONGJMP: return "PPCISD::EH_SJLJ_LONGJMP"; 1656 case PPCISD::MFOCRF: return "PPCISD::MFOCRF"; 1657 case PPCISD::MFVSR: return "PPCISD::MFVSR"; 1658 case PPCISD::MTVSRA: return "PPCISD::MTVSRA"; 1659 case PPCISD::MTVSRZ: return "PPCISD::MTVSRZ"; 1660 case PPCISD::SINT_VEC_TO_FP: return "PPCISD::SINT_VEC_TO_FP"; 1661 case PPCISD::UINT_VEC_TO_FP: return "PPCISD::UINT_VEC_TO_FP"; 1662 case PPCISD::SCALAR_TO_VECTOR_PERMUTED: 1663 return "PPCISD::SCALAR_TO_VECTOR_PERMUTED"; 1664 case PPCISD::ANDI_rec_1_EQ_BIT: 1665 return "PPCISD::ANDI_rec_1_EQ_BIT"; 1666 case PPCISD::ANDI_rec_1_GT_BIT: 1667 return "PPCISD::ANDI_rec_1_GT_BIT"; 1668 case PPCISD::VCMP: return "PPCISD::VCMP"; 1669 case PPCISD::VCMP_rec: return "PPCISD::VCMP_rec"; 1670 case PPCISD::LBRX: return "PPCISD::LBRX"; 1671 case PPCISD::STBRX: return "PPCISD::STBRX"; 1672 case PPCISD::LFIWAX: return "PPCISD::LFIWAX"; 1673 case PPCISD::LFIWZX: return "PPCISD::LFIWZX"; 1674 case PPCISD::LXSIZX: return "PPCISD::LXSIZX"; 1675 case PPCISD::STXSIX: return "PPCISD::STXSIX"; 1676 case PPCISD::VEXTS: return "PPCISD::VEXTS"; 1677 case PPCISD::LXVD2X: return "PPCISD::LXVD2X"; 1678 case PPCISD::STXVD2X: return "PPCISD::STXVD2X"; 1679 case PPCISD::LOAD_VEC_BE: return "PPCISD::LOAD_VEC_BE"; 1680 case PPCISD::STORE_VEC_BE: return "PPCISD::STORE_VEC_BE"; 1681 case PPCISD::ST_VSR_SCAL_INT: 1682 return "PPCISD::ST_VSR_SCAL_INT"; 1683 case PPCISD::COND_BRANCH: return "PPCISD::COND_BRANCH"; 1684 case PPCISD::BDNZ: return "PPCISD::BDNZ"; 1685 case PPCISD::BDZ: return "PPCISD::BDZ"; 1686 case PPCISD::MFFS: return "PPCISD::MFFS"; 1687 case PPCISD::FADDRTZ: return "PPCISD::FADDRTZ"; 1688 case PPCISD::TC_RETURN: return "PPCISD::TC_RETURN"; 1689 case PPCISD::CR6SET: return "PPCISD::CR6SET"; 1690 case PPCISD::CR6UNSET: return "PPCISD::CR6UNSET"; 1691 case PPCISD::PPC32_GOT: return "PPCISD::PPC32_GOT"; 1692 case PPCISD::PPC32_PICGOT: return "PPCISD::PPC32_PICGOT"; 1693 case PPCISD::ADDIS_GOT_TPREL_HA: return "PPCISD::ADDIS_GOT_TPREL_HA"; 1694 case PPCISD::LD_GOT_TPREL_L: return "PPCISD::LD_GOT_TPREL_L"; 1695 case PPCISD::ADD_TLS: return "PPCISD::ADD_TLS"; 1696 case PPCISD::ADDIS_TLSGD_HA: return "PPCISD::ADDIS_TLSGD_HA"; 1697 case PPCISD::ADDI_TLSGD_L: return "PPCISD::ADDI_TLSGD_L"; 1698 case PPCISD::GET_TLS_ADDR: return "PPCISD::GET_TLS_ADDR"; 1699 case PPCISD::ADDI_TLSGD_L_ADDR: return "PPCISD::ADDI_TLSGD_L_ADDR"; 1700 case PPCISD::TLSGD_AIX: return "PPCISD::TLSGD_AIX"; 1701 case PPCISD::ADDIS_TLSLD_HA: return "PPCISD::ADDIS_TLSLD_HA"; 1702 case PPCISD::ADDI_TLSLD_L: return "PPCISD::ADDI_TLSLD_L"; 1703 case PPCISD::GET_TLSLD_ADDR: return "PPCISD::GET_TLSLD_ADDR"; 1704 case PPCISD::ADDI_TLSLD_L_ADDR: return "PPCISD::ADDI_TLSLD_L_ADDR"; 1705 case PPCISD::ADDIS_DTPREL_HA: return "PPCISD::ADDIS_DTPREL_HA"; 1706 case PPCISD::ADDI_DTPREL_L: return "PPCISD::ADDI_DTPREL_L"; 1707 case PPCISD::PADDI_DTPREL: 1708 return "PPCISD::PADDI_DTPREL"; 1709 case PPCISD::VADD_SPLAT: return "PPCISD::VADD_SPLAT"; 1710 case PPCISD::SC: return "PPCISD::SC"; 1711 case PPCISD::CLRBHRB: return "PPCISD::CLRBHRB"; 1712 case PPCISD::MFBHRBE: return "PPCISD::MFBHRBE"; 1713 case PPCISD::RFEBB: return "PPCISD::RFEBB"; 1714 case PPCISD::XXSWAPD: return "PPCISD::XXSWAPD"; 1715 case PPCISD::SWAP_NO_CHAIN: return "PPCISD::SWAP_NO_CHAIN"; 1716 case PPCISD::VABSD: return "PPCISD::VABSD"; 1717 case PPCISD::BUILD_FP128: return "PPCISD::BUILD_FP128"; 1718 case PPCISD::BUILD_SPE64: return "PPCISD::BUILD_SPE64"; 1719 case PPCISD::EXTRACT_SPE: return "PPCISD::EXTRACT_SPE"; 1720 case PPCISD::EXTSWSLI: return "PPCISD::EXTSWSLI"; 1721 case PPCISD::LD_VSX_LH: return "PPCISD::LD_VSX_LH"; 1722 case PPCISD::FP_EXTEND_HALF: return "PPCISD::FP_EXTEND_HALF"; 1723 case PPCISD::MAT_PCREL_ADDR: return "PPCISD::MAT_PCREL_ADDR"; 1724 case PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR: 1725 return "PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR"; 1726 case PPCISD::TLS_LOCAL_EXEC_MAT_ADDR: 1727 return "PPCISD::TLS_LOCAL_EXEC_MAT_ADDR"; 1728 case PPCISD::ACC_BUILD: return "PPCISD::ACC_BUILD"; 1729 case PPCISD::PAIR_BUILD: return "PPCISD::PAIR_BUILD"; 1730 case PPCISD::EXTRACT_VSX_REG: return "PPCISD::EXTRACT_VSX_REG"; 1731 case PPCISD::XXMFACC: return "PPCISD::XXMFACC"; 1732 case PPCISD::LD_SPLAT: return "PPCISD::LD_SPLAT"; 1733 case PPCISD::ZEXT_LD_SPLAT: return "PPCISD::ZEXT_LD_SPLAT"; 1734 case PPCISD::SEXT_LD_SPLAT: return "PPCISD::SEXT_LD_SPLAT"; 1735 case PPCISD::FNMSUB: return "PPCISD::FNMSUB"; 1736 case PPCISD::STRICT_FADDRTZ: 1737 return "PPCISD::STRICT_FADDRTZ"; 1738 case PPCISD::STRICT_FCTIDZ: 1739 return "PPCISD::STRICT_FCTIDZ"; 1740 case PPCISD::STRICT_FCTIWZ: 1741 return "PPCISD::STRICT_FCTIWZ"; 1742 case PPCISD::STRICT_FCTIDUZ: 1743 return "PPCISD::STRICT_FCTIDUZ"; 1744 case PPCISD::STRICT_FCTIWUZ: 1745 return "PPCISD::STRICT_FCTIWUZ"; 1746 case PPCISD::STRICT_FCFID: 1747 return "PPCISD::STRICT_FCFID"; 1748 case PPCISD::STRICT_FCFIDU: 1749 return "PPCISD::STRICT_FCFIDU"; 1750 case PPCISD::STRICT_FCFIDS: 1751 return "PPCISD::STRICT_FCFIDS"; 1752 case PPCISD::STRICT_FCFIDUS: 1753 return "PPCISD::STRICT_FCFIDUS"; 1754 case PPCISD::LXVRZX: return "PPCISD::LXVRZX"; 1755 } 1756 return nullptr; 1757 } 1758 1759 EVT PPCTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C, 1760 EVT VT) const { 1761 if (!VT.isVector()) 1762 return Subtarget.useCRBits() ? MVT::i1 : MVT::i32; 1763 1764 return VT.changeVectorElementTypeToInteger(); 1765 } 1766 1767 bool PPCTargetLowering::enableAggressiveFMAFusion(EVT VT) const { 1768 assert(VT.isFloatingPoint() && "Non-floating-point FMA?"); 1769 return true; 1770 } 1771 1772 //===----------------------------------------------------------------------===// 1773 // Node matching predicates, for use by the tblgen matching code. 1774 //===----------------------------------------------------------------------===// 1775 1776 /// isFloatingPointZero - Return true if this is 0.0 or -0.0. 1777 static bool isFloatingPointZero(SDValue Op) { 1778 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Op)) 1779 return CFP->getValueAPF().isZero(); 1780 else if (ISD::isEXTLoad(Op.getNode()) || ISD::isNON_EXTLoad(Op.getNode())) { 1781 // Maybe this has already been legalized into the constant pool? 1782 if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Op.getOperand(1))) 1783 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal())) 1784 return CFP->getValueAPF().isZero(); 1785 } 1786 return false; 1787 } 1788 1789 /// isConstantOrUndef - Op is either an undef node or a ConstantSDNode. Return 1790 /// true if Op is undef or if it matches the specified value. 1791 static bool isConstantOrUndef(int Op, int Val) { 1792 return Op < 0 || Op == Val; 1793 } 1794 1795 /// isVPKUHUMShuffleMask - Return true if this is the shuffle mask for a 1796 /// VPKUHUM instruction. 1797 /// The ShuffleKind distinguishes between big-endian operations with 1798 /// two different inputs (0), either-endian operations with two identical 1799 /// inputs (1), and little-endian operations with two different inputs (2). 1800 /// For the latter, the input operands are swapped (see PPCInstrAltivec.td). 1801 bool PPC::isVPKUHUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind, 1802 SelectionDAG &DAG) { 1803 bool IsLE = DAG.getDataLayout().isLittleEndian(); 1804 if (ShuffleKind == 0) { 1805 if (IsLE) 1806 return false; 1807 for (unsigned i = 0; i != 16; ++i) 1808 if (!isConstantOrUndef(N->getMaskElt(i), i*2+1)) 1809 return false; 1810 } else if (ShuffleKind == 2) { 1811 if (!IsLE) 1812 return false; 1813 for (unsigned i = 0; i != 16; ++i) 1814 if (!isConstantOrUndef(N->getMaskElt(i), i*2)) 1815 return false; 1816 } else if (ShuffleKind == 1) { 1817 unsigned j = IsLE ? 0 : 1; 1818 for (unsigned i = 0; i != 8; ++i) 1819 if (!isConstantOrUndef(N->getMaskElt(i), i*2+j) || 1820 !isConstantOrUndef(N->getMaskElt(i+8), i*2+j)) 1821 return false; 1822 } 1823 return true; 1824 } 1825 1826 /// isVPKUWUMShuffleMask - Return true if this is the shuffle mask for a 1827 /// VPKUWUM instruction. 1828 /// The ShuffleKind distinguishes between big-endian operations with 1829 /// two different inputs (0), either-endian operations with two identical 1830 /// inputs (1), and little-endian operations with two different inputs (2). 1831 /// For the latter, the input operands are swapped (see PPCInstrAltivec.td). 1832 bool PPC::isVPKUWUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind, 1833 SelectionDAG &DAG) { 1834 bool IsLE = DAG.getDataLayout().isLittleEndian(); 1835 if (ShuffleKind == 0) { 1836 if (IsLE) 1837 return false; 1838 for (unsigned i = 0; i != 16; i += 2) 1839 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+2) || 1840 !isConstantOrUndef(N->getMaskElt(i+1), i*2+3)) 1841 return false; 1842 } else if (ShuffleKind == 2) { 1843 if (!IsLE) 1844 return false; 1845 for (unsigned i = 0; i != 16; i += 2) 1846 if (!isConstantOrUndef(N->getMaskElt(i ), i*2) || 1847 !isConstantOrUndef(N->getMaskElt(i+1), i*2+1)) 1848 return false; 1849 } else if (ShuffleKind == 1) { 1850 unsigned j = IsLE ? 0 : 2; 1851 for (unsigned i = 0; i != 8; i += 2) 1852 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+j) || 1853 !isConstantOrUndef(N->getMaskElt(i+1), i*2+j+1) || 1854 !isConstantOrUndef(N->getMaskElt(i+8), i*2+j) || 1855 !isConstantOrUndef(N->getMaskElt(i+9), i*2+j+1)) 1856 return false; 1857 } 1858 return true; 1859 } 1860 1861 /// isVPKUDUMShuffleMask - Return true if this is the shuffle mask for a 1862 /// VPKUDUM instruction, AND the VPKUDUM instruction exists for the 1863 /// current subtarget. 1864 /// 1865 /// The ShuffleKind distinguishes between big-endian operations with 1866 /// two different inputs (0), either-endian operations with two identical 1867 /// inputs (1), and little-endian operations with two different inputs (2). 1868 /// For the latter, the input operands are swapped (see PPCInstrAltivec.td). 1869 bool PPC::isVPKUDUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind, 1870 SelectionDAG &DAG) { 1871 const PPCSubtarget& Subtarget = 1872 static_cast<const PPCSubtarget&>(DAG.getSubtarget()); 1873 if (!Subtarget.hasP8Vector()) 1874 return false; 1875 1876 bool IsLE = DAG.getDataLayout().isLittleEndian(); 1877 if (ShuffleKind == 0) { 1878 if (IsLE) 1879 return false; 1880 for (unsigned i = 0; i != 16; i += 4) 1881 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+4) || 1882 !isConstantOrUndef(N->getMaskElt(i+1), i*2+5) || 1883 !isConstantOrUndef(N->getMaskElt(i+2), i*2+6) || 1884 !isConstantOrUndef(N->getMaskElt(i+3), i*2+7)) 1885 return false; 1886 } else if (ShuffleKind == 2) { 1887 if (!IsLE) 1888 return false; 1889 for (unsigned i = 0; i != 16; i += 4) 1890 if (!isConstantOrUndef(N->getMaskElt(i ), i*2) || 1891 !isConstantOrUndef(N->getMaskElt(i+1), i*2+1) || 1892 !isConstantOrUndef(N->getMaskElt(i+2), i*2+2) || 1893 !isConstantOrUndef(N->getMaskElt(i+3), i*2+3)) 1894 return false; 1895 } else if (ShuffleKind == 1) { 1896 unsigned j = IsLE ? 0 : 4; 1897 for (unsigned i = 0; i != 8; i += 4) 1898 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+j) || 1899 !isConstantOrUndef(N->getMaskElt(i+1), i*2+j+1) || 1900 !isConstantOrUndef(N->getMaskElt(i+2), i*2+j+2) || 1901 !isConstantOrUndef(N->getMaskElt(i+3), i*2+j+3) || 1902 !isConstantOrUndef(N->getMaskElt(i+8), i*2+j) || 1903 !isConstantOrUndef(N->getMaskElt(i+9), i*2+j+1) || 1904 !isConstantOrUndef(N->getMaskElt(i+10), i*2+j+2) || 1905 !isConstantOrUndef(N->getMaskElt(i+11), i*2+j+3)) 1906 return false; 1907 } 1908 return true; 1909 } 1910 1911 /// isVMerge - Common function, used to match vmrg* shuffles. 1912 /// 1913 static bool isVMerge(ShuffleVectorSDNode *N, unsigned UnitSize, 1914 unsigned LHSStart, unsigned RHSStart) { 1915 if (N->getValueType(0) != MVT::v16i8) 1916 return false; 1917 assert((UnitSize == 1 || UnitSize == 2 || UnitSize == 4) && 1918 "Unsupported merge size!"); 1919 1920 for (unsigned i = 0; i != 8/UnitSize; ++i) // Step over units 1921 for (unsigned j = 0; j != UnitSize; ++j) { // Step over bytes within unit 1922 if (!isConstantOrUndef(N->getMaskElt(i*UnitSize*2+j), 1923 LHSStart+j+i*UnitSize) || 1924 !isConstantOrUndef(N->getMaskElt(i*UnitSize*2+UnitSize+j), 1925 RHSStart+j+i*UnitSize)) 1926 return false; 1927 } 1928 return true; 1929 } 1930 1931 /// isVMRGLShuffleMask - Return true if this is a shuffle mask suitable for 1932 /// a VMRGL* instruction with the specified unit size (1,2 or 4 bytes). 1933 /// The ShuffleKind distinguishes between big-endian merges with two 1934 /// different inputs (0), either-endian merges with two identical inputs (1), 1935 /// and little-endian merges with two different inputs (2). For the latter, 1936 /// the input operands are swapped (see PPCInstrAltivec.td). 1937 bool PPC::isVMRGLShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize, 1938 unsigned ShuffleKind, SelectionDAG &DAG) { 1939 if (DAG.getDataLayout().isLittleEndian()) { 1940 if (ShuffleKind == 1) // unary 1941 return isVMerge(N, UnitSize, 0, 0); 1942 else if (ShuffleKind == 2) // swapped 1943 return isVMerge(N, UnitSize, 0, 16); 1944 else 1945 return false; 1946 } else { 1947 if (ShuffleKind == 1) // unary 1948 return isVMerge(N, UnitSize, 8, 8); 1949 else if (ShuffleKind == 0) // normal 1950 return isVMerge(N, UnitSize, 8, 24); 1951 else 1952 return false; 1953 } 1954 } 1955 1956 /// isVMRGHShuffleMask - Return true if this is a shuffle mask suitable for 1957 /// a VMRGH* instruction with the specified unit size (1,2 or 4 bytes). 1958 /// The ShuffleKind distinguishes between big-endian merges with two 1959 /// different inputs (0), either-endian merges with two identical inputs (1), 1960 /// and little-endian merges with two different inputs (2). For the latter, 1961 /// the input operands are swapped (see PPCInstrAltivec.td). 1962 bool PPC::isVMRGHShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize, 1963 unsigned ShuffleKind, SelectionDAG &DAG) { 1964 if (DAG.getDataLayout().isLittleEndian()) { 1965 if (ShuffleKind == 1) // unary 1966 return isVMerge(N, UnitSize, 8, 8); 1967 else if (ShuffleKind == 2) // swapped 1968 return isVMerge(N, UnitSize, 8, 24); 1969 else 1970 return false; 1971 } else { 1972 if (ShuffleKind == 1) // unary 1973 return isVMerge(N, UnitSize, 0, 0); 1974 else if (ShuffleKind == 0) // normal 1975 return isVMerge(N, UnitSize, 0, 16); 1976 else 1977 return false; 1978 } 1979 } 1980 1981 /** 1982 * Common function used to match vmrgew and vmrgow shuffles 1983 * 1984 * The indexOffset determines whether to look for even or odd words in 1985 * the shuffle mask. This is based on the of the endianness of the target 1986 * machine. 1987 * - Little Endian: 1988 * - Use offset of 0 to check for odd elements 1989 * - Use offset of 4 to check for even elements 1990 * - Big Endian: 1991 * - Use offset of 0 to check for even elements 1992 * - Use offset of 4 to check for odd elements 1993 * A detailed description of the vector element ordering for little endian and 1994 * big endian can be found at 1995 * http://www.ibm.com/developerworks/library/l-ibm-xl-c-cpp-compiler/index.html 1996 * Targeting your applications - what little endian and big endian IBM XL C/C++ 1997 * compiler differences mean to you 1998 * 1999 * The mask to the shuffle vector instruction specifies the indices of the 2000 * elements from the two input vectors to place in the result. The elements are 2001 * numbered in array-access order, starting with the first vector. These vectors 2002 * are always of type v16i8, thus each vector will contain 16 elements of size 2003 * 8. More info on the shuffle vector can be found in the 2004 * http://llvm.org/docs/LangRef.html#shufflevector-instruction 2005 * Language Reference. 2006 * 2007 * The RHSStartValue indicates whether the same input vectors are used (unary) 2008 * or two different input vectors are used, based on the following: 2009 * - If the instruction uses the same vector for both inputs, the range of the 2010 * indices will be 0 to 15. In this case, the RHSStart value passed should 2011 * be 0. 2012 * - If the instruction has two different vectors then the range of the 2013 * indices will be 0 to 31. In this case, the RHSStart value passed should 2014 * be 16 (indices 0-15 specify elements in the first vector while indices 16 2015 * to 31 specify elements in the second vector). 2016 * 2017 * \param[in] N The shuffle vector SD Node to analyze 2018 * \param[in] IndexOffset Specifies whether to look for even or odd elements 2019 * \param[in] RHSStartValue Specifies the starting index for the righthand input 2020 * vector to the shuffle_vector instruction 2021 * \return true iff this shuffle vector represents an even or odd word merge 2022 */ 2023 static bool isVMerge(ShuffleVectorSDNode *N, unsigned IndexOffset, 2024 unsigned RHSStartValue) { 2025 if (N->getValueType(0) != MVT::v16i8) 2026 return false; 2027 2028 for (unsigned i = 0; i < 2; ++i) 2029 for (unsigned j = 0; j < 4; ++j) 2030 if (!isConstantOrUndef(N->getMaskElt(i*4+j), 2031 i*RHSStartValue+j+IndexOffset) || 2032 !isConstantOrUndef(N->getMaskElt(i*4+j+8), 2033 i*RHSStartValue+j+IndexOffset+8)) 2034 return false; 2035 return true; 2036 } 2037 2038 /** 2039 * Determine if the specified shuffle mask is suitable for the vmrgew or 2040 * vmrgow instructions. 2041 * 2042 * \param[in] N The shuffle vector SD Node to analyze 2043 * \param[in] CheckEven Check for an even merge (true) or an odd merge (false) 2044 * \param[in] ShuffleKind Identify the type of merge: 2045 * - 0 = big-endian merge with two different inputs; 2046 * - 1 = either-endian merge with two identical inputs; 2047 * - 2 = little-endian merge with two different inputs (inputs are swapped for 2048 * little-endian merges). 2049 * \param[in] DAG The current SelectionDAG 2050 * \return true iff this shuffle mask 2051 */ 2052 bool PPC::isVMRGEOShuffleMask(ShuffleVectorSDNode *N, bool CheckEven, 2053 unsigned ShuffleKind, SelectionDAG &DAG) { 2054 if (DAG.getDataLayout().isLittleEndian()) { 2055 unsigned indexOffset = CheckEven ? 4 : 0; 2056 if (ShuffleKind == 1) // Unary 2057 return isVMerge(N, indexOffset, 0); 2058 else if (ShuffleKind == 2) // swapped 2059 return isVMerge(N, indexOffset, 16); 2060 else 2061 return false; 2062 } 2063 else { 2064 unsigned indexOffset = CheckEven ? 0 : 4; 2065 if (ShuffleKind == 1) // Unary 2066 return isVMerge(N, indexOffset, 0); 2067 else if (ShuffleKind == 0) // Normal 2068 return isVMerge(N, indexOffset, 16); 2069 else 2070 return false; 2071 } 2072 return false; 2073 } 2074 2075 /// isVSLDOIShuffleMask - If this is a vsldoi shuffle mask, return the shift 2076 /// amount, otherwise return -1. 2077 /// The ShuffleKind distinguishes between big-endian operations with two 2078 /// different inputs (0), either-endian operations with two identical inputs 2079 /// (1), and little-endian operations with two different inputs (2). For the 2080 /// latter, the input operands are swapped (see PPCInstrAltivec.td). 2081 int PPC::isVSLDOIShuffleMask(SDNode *N, unsigned ShuffleKind, 2082 SelectionDAG &DAG) { 2083 if (N->getValueType(0) != MVT::v16i8) 2084 return -1; 2085 2086 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 2087 2088 // Find the first non-undef value in the shuffle mask. 2089 unsigned i; 2090 for (i = 0; i != 16 && SVOp->getMaskElt(i) < 0; ++i) 2091 /*search*/; 2092 2093 if (i == 16) return -1; // all undef. 2094 2095 // Otherwise, check to see if the rest of the elements are consecutively 2096 // numbered from this value. 2097 unsigned ShiftAmt = SVOp->getMaskElt(i); 2098 if (ShiftAmt < i) return -1; 2099 2100 ShiftAmt -= i; 2101 bool isLE = DAG.getDataLayout().isLittleEndian(); 2102 2103 if ((ShuffleKind == 0 && !isLE) || (ShuffleKind == 2 && isLE)) { 2104 // Check the rest of the elements to see if they are consecutive. 2105 for (++i; i != 16; ++i) 2106 if (!isConstantOrUndef(SVOp->getMaskElt(i), ShiftAmt+i)) 2107 return -1; 2108 } else if (ShuffleKind == 1) { 2109 // Check the rest of the elements to see if they are consecutive. 2110 for (++i; i != 16; ++i) 2111 if (!isConstantOrUndef(SVOp->getMaskElt(i), (ShiftAmt+i) & 15)) 2112 return -1; 2113 } else 2114 return -1; 2115 2116 if (isLE) 2117 ShiftAmt = 16 - ShiftAmt; 2118 2119 return ShiftAmt; 2120 } 2121 2122 /// isSplatShuffleMask - Return true if the specified VECTOR_SHUFFLE operand 2123 /// specifies a splat of a single element that is suitable for input to 2124 /// one of the splat operations (VSPLTB/VSPLTH/VSPLTW/XXSPLTW/LXVDSX/etc.). 2125 bool PPC::isSplatShuffleMask(ShuffleVectorSDNode *N, unsigned EltSize) { 2126 assert(N->getValueType(0) == MVT::v16i8 && isPowerOf2_32(EltSize) && 2127 EltSize <= 8 && "Can only handle 1,2,4,8 byte element sizes"); 2128 2129 // The consecutive indices need to specify an element, not part of two 2130 // different elements. So abandon ship early if this isn't the case. 2131 if (N->getMaskElt(0) % EltSize != 0) 2132 return false; 2133 2134 // This is a splat operation if each element of the permute is the same, and 2135 // if the value doesn't reference the second vector. 2136 unsigned ElementBase = N->getMaskElt(0); 2137 2138 // FIXME: Handle UNDEF elements too! 2139 if (ElementBase >= 16) 2140 return false; 2141 2142 // Check that the indices are consecutive, in the case of a multi-byte element 2143 // splatted with a v16i8 mask. 2144 for (unsigned i = 1; i != EltSize; ++i) 2145 if (N->getMaskElt(i) < 0 || N->getMaskElt(i) != (int)(i+ElementBase)) 2146 return false; 2147 2148 for (unsigned i = EltSize, e = 16; i != e; i += EltSize) { 2149 if (N->getMaskElt(i) < 0) continue; 2150 for (unsigned j = 0; j != EltSize; ++j) 2151 if (N->getMaskElt(i+j) != N->getMaskElt(j)) 2152 return false; 2153 } 2154 return true; 2155 } 2156 2157 /// Check that the mask is shuffling N byte elements. Within each N byte 2158 /// element of the mask, the indices could be either in increasing or 2159 /// decreasing order as long as they are consecutive. 2160 /// \param[in] N the shuffle vector SD Node to analyze 2161 /// \param[in] Width the element width in bytes, could be 2/4/8/16 (HalfWord/ 2162 /// Word/DoubleWord/QuadWord). 2163 /// \param[in] StepLen the delta indices number among the N byte element, if 2164 /// the mask is in increasing/decreasing order then it is 1/-1. 2165 /// \return true iff the mask is shuffling N byte elements. 2166 static bool isNByteElemShuffleMask(ShuffleVectorSDNode *N, unsigned Width, 2167 int StepLen) { 2168 assert((Width == 2 || Width == 4 || Width == 8 || Width == 16) && 2169 "Unexpected element width."); 2170 assert((StepLen == 1 || StepLen == -1) && "Unexpected element width."); 2171 2172 unsigned NumOfElem = 16 / Width; 2173 unsigned MaskVal[16]; // Width is never greater than 16 2174 for (unsigned i = 0; i < NumOfElem; ++i) { 2175 MaskVal[0] = N->getMaskElt(i * Width); 2176 if ((StepLen == 1) && (MaskVal[0] % Width)) { 2177 return false; 2178 } else if ((StepLen == -1) && ((MaskVal[0] + 1) % Width)) { 2179 return false; 2180 } 2181 2182 for (unsigned int j = 1; j < Width; ++j) { 2183 MaskVal[j] = N->getMaskElt(i * Width + j); 2184 if (MaskVal[j] != MaskVal[j-1] + StepLen) { 2185 return false; 2186 } 2187 } 2188 } 2189 2190 return true; 2191 } 2192 2193 bool PPC::isXXINSERTWMask(ShuffleVectorSDNode *N, unsigned &ShiftElts, 2194 unsigned &InsertAtByte, bool &Swap, bool IsLE) { 2195 if (!isNByteElemShuffleMask(N, 4, 1)) 2196 return false; 2197 2198 // Now we look at mask elements 0,4,8,12 2199 unsigned M0 = N->getMaskElt(0) / 4; 2200 unsigned M1 = N->getMaskElt(4) / 4; 2201 unsigned M2 = N->getMaskElt(8) / 4; 2202 unsigned M3 = N->getMaskElt(12) / 4; 2203 unsigned LittleEndianShifts[] = { 2, 1, 0, 3 }; 2204 unsigned BigEndianShifts[] = { 3, 0, 1, 2 }; 2205 2206 // Below, let H and L be arbitrary elements of the shuffle mask 2207 // where H is in the range [4,7] and L is in the range [0,3]. 2208 // H, 1, 2, 3 or L, 5, 6, 7 2209 if ((M0 > 3 && M1 == 1 && M2 == 2 && M3 == 3) || 2210 (M0 < 4 && M1 == 5 && M2 == 6 && M3 == 7)) { 2211 ShiftElts = IsLE ? LittleEndianShifts[M0 & 0x3] : BigEndianShifts[M0 & 0x3]; 2212 InsertAtByte = IsLE ? 12 : 0; 2213 Swap = M0 < 4; 2214 return true; 2215 } 2216 // 0, H, 2, 3 or 4, L, 6, 7 2217 if ((M1 > 3 && M0 == 0 && M2 == 2 && M3 == 3) || 2218 (M1 < 4 && M0 == 4 && M2 == 6 && M3 == 7)) { 2219 ShiftElts = IsLE ? LittleEndianShifts[M1 & 0x3] : BigEndianShifts[M1 & 0x3]; 2220 InsertAtByte = IsLE ? 8 : 4; 2221 Swap = M1 < 4; 2222 return true; 2223 } 2224 // 0, 1, H, 3 or 4, 5, L, 7 2225 if ((M2 > 3 && M0 == 0 && M1 == 1 && M3 == 3) || 2226 (M2 < 4 && M0 == 4 && M1 == 5 && M3 == 7)) { 2227 ShiftElts = IsLE ? LittleEndianShifts[M2 & 0x3] : BigEndianShifts[M2 & 0x3]; 2228 InsertAtByte = IsLE ? 4 : 8; 2229 Swap = M2 < 4; 2230 return true; 2231 } 2232 // 0, 1, 2, H or 4, 5, 6, L 2233 if ((M3 > 3 && M0 == 0 && M1 == 1 && M2 == 2) || 2234 (M3 < 4 && M0 == 4 && M1 == 5 && M2 == 6)) { 2235 ShiftElts = IsLE ? LittleEndianShifts[M3 & 0x3] : BigEndianShifts[M3 & 0x3]; 2236 InsertAtByte = IsLE ? 0 : 12; 2237 Swap = M3 < 4; 2238 return true; 2239 } 2240 2241 // If both vector operands for the shuffle are the same vector, the mask will 2242 // contain only elements from the first one and the second one will be undef. 2243 if (N->getOperand(1).isUndef()) { 2244 ShiftElts = 0; 2245 Swap = true; 2246 unsigned XXINSERTWSrcElem = IsLE ? 2 : 1; 2247 if (M0 == XXINSERTWSrcElem && M1 == 1 && M2 == 2 && M3 == 3) { 2248 InsertAtByte = IsLE ? 12 : 0; 2249 return true; 2250 } 2251 if (M0 == 0 && M1 == XXINSERTWSrcElem && M2 == 2 && M3 == 3) { 2252 InsertAtByte = IsLE ? 8 : 4; 2253 return true; 2254 } 2255 if (M0 == 0 && M1 == 1 && M2 == XXINSERTWSrcElem && M3 == 3) { 2256 InsertAtByte = IsLE ? 4 : 8; 2257 return true; 2258 } 2259 if (M0 == 0 && M1 == 1 && M2 == 2 && M3 == XXINSERTWSrcElem) { 2260 InsertAtByte = IsLE ? 0 : 12; 2261 return true; 2262 } 2263 } 2264 2265 return false; 2266 } 2267 2268 bool PPC::isXXSLDWIShuffleMask(ShuffleVectorSDNode *N, unsigned &ShiftElts, 2269 bool &Swap, bool IsLE) { 2270 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8"); 2271 // Ensure each byte index of the word is consecutive. 2272 if (!isNByteElemShuffleMask(N, 4, 1)) 2273 return false; 2274 2275 // Now we look at mask elements 0,4,8,12, which are the beginning of words. 2276 unsigned M0 = N->getMaskElt(0) / 4; 2277 unsigned M1 = N->getMaskElt(4) / 4; 2278 unsigned M2 = N->getMaskElt(8) / 4; 2279 unsigned M3 = N->getMaskElt(12) / 4; 2280 2281 // If both vector operands for the shuffle are the same vector, the mask will 2282 // contain only elements from the first one and the second one will be undef. 2283 if (N->getOperand(1).isUndef()) { 2284 assert(M0 < 4 && "Indexing into an undef vector?"); 2285 if (M1 != (M0 + 1) % 4 || M2 != (M1 + 1) % 4 || M3 != (M2 + 1) % 4) 2286 return false; 2287 2288 ShiftElts = IsLE ? (4 - M0) % 4 : M0; 2289 Swap = false; 2290 return true; 2291 } 2292 2293 // Ensure each word index of the ShuffleVector Mask is consecutive. 2294 if (M1 != (M0 + 1) % 8 || M2 != (M1 + 1) % 8 || M3 != (M2 + 1) % 8) 2295 return false; 2296 2297 if (IsLE) { 2298 if (M0 == 0 || M0 == 7 || M0 == 6 || M0 == 5) { 2299 // Input vectors don't need to be swapped if the leading element 2300 // of the result is one of the 3 left elements of the second vector 2301 // (or if there is no shift to be done at all). 2302 Swap = false; 2303 ShiftElts = (8 - M0) % 8; 2304 } else if (M0 == 4 || M0 == 3 || M0 == 2 || M0 == 1) { 2305 // Input vectors need to be swapped if the leading element 2306 // of the result is one of the 3 left elements of the first vector 2307 // (or if we're shifting by 4 - thereby simply swapping the vectors). 2308 Swap = true; 2309 ShiftElts = (4 - M0) % 4; 2310 } 2311 2312 return true; 2313 } else { // BE 2314 if (M0 == 0 || M0 == 1 || M0 == 2 || M0 == 3) { 2315 // Input vectors don't need to be swapped if the leading element 2316 // of the result is one of the 4 elements of the first vector. 2317 Swap = false; 2318 ShiftElts = M0; 2319 } else if (M0 == 4 || M0 == 5 || M0 == 6 || M0 == 7) { 2320 // Input vectors need to be swapped if the leading element 2321 // of the result is one of the 4 elements of the right vector. 2322 Swap = true; 2323 ShiftElts = M0 - 4; 2324 } 2325 2326 return true; 2327 } 2328 } 2329 2330 bool static isXXBRShuffleMaskHelper(ShuffleVectorSDNode *N, int Width) { 2331 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8"); 2332 2333 if (!isNByteElemShuffleMask(N, Width, -1)) 2334 return false; 2335 2336 for (int i = 0; i < 16; i += Width) 2337 if (N->getMaskElt(i) != i + Width - 1) 2338 return false; 2339 2340 return true; 2341 } 2342 2343 bool PPC::isXXBRHShuffleMask(ShuffleVectorSDNode *N) { 2344 return isXXBRShuffleMaskHelper(N, 2); 2345 } 2346 2347 bool PPC::isXXBRWShuffleMask(ShuffleVectorSDNode *N) { 2348 return isXXBRShuffleMaskHelper(N, 4); 2349 } 2350 2351 bool PPC::isXXBRDShuffleMask(ShuffleVectorSDNode *N) { 2352 return isXXBRShuffleMaskHelper(N, 8); 2353 } 2354 2355 bool PPC::isXXBRQShuffleMask(ShuffleVectorSDNode *N) { 2356 return isXXBRShuffleMaskHelper(N, 16); 2357 } 2358 2359 /// Can node \p N be lowered to an XXPERMDI instruction? If so, set \p Swap 2360 /// if the inputs to the instruction should be swapped and set \p DM to the 2361 /// value for the immediate. 2362 /// Specifically, set \p Swap to true only if \p N can be lowered to XXPERMDI 2363 /// AND element 0 of the result comes from the first input (LE) or second input 2364 /// (BE). Set \p DM to the calculated result (0-3) only if \p N can be lowered. 2365 /// \return true iff the given mask of shuffle node \p N is a XXPERMDI shuffle 2366 /// mask. 2367 bool PPC::isXXPERMDIShuffleMask(ShuffleVectorSDNode *N, unsigned &DM, 2368 bool &Swap, bool IsLE) { 2369 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8"); 2370 2371 // Ensure each byte index of the double word is consecutive. 2372 if (!isNByteElemShuffleMask(N, 8, 1)) 2373 return false; 2374 2375 unsigned M0 = N->getMaskElt(0) / 8; 2376 unsigned M1 = N->getMaskElt(8) / 8; 2377 assert(((M0 | M1) < 4) && "A mask element out of bounds?"); 2378 2379 // If both vector operands for the shuffle are the same vector, the mask will 2380 // contain only elements from the first one and the second one will be undef. 2381 if (N->getOperand(1).isUndef()) { 2382 if ((M0 | M1) < 2) { 2383 DM = IsLE ? (((~M1) & 1) << 1) + ((~M0) & 1) : (M0 << 1) + (M1 & 1); 2384 Swap = false; 2385 return true; 2386 } else 2387 return false; 2388 } 2389 2390 if (IsLE) { 2391 if (M0 > 1 && M1 < 2) { 2392 Swap = false; 2393 } else if (M0 < 2 && M1 > 1) { 2394 M0 = (M0 + 2) % 4; 2395 M1 = (M1 + 2) % 4; 2396 Swap = true; 2397 } else 2398 return false; 2399 2400 // Note: if control flow comes here that means Swap is already set above 2401 DM = (((~M1) & 1) << 1) + ((~M0) & 1); 2402 return true; 2403 } else { // BE 2404 if (M0 < 2 && M1 > 1) { 2405 Swap = false; 2406 } else if (M0 > 1 && M1 < 2) { 2407 M0 = (M0 + 2) % 4; 2408 M1 = (M1 + 2) % 4; 2409 Swap = true; 2410 } else 2411 return false; 2412 2413 // Note: if control flow comes here that means Swap is already set above 2414 DM = (M0 << 1) + (M1 & 1); 2415 return true; 2416 } 2417 } 2418 2419 2420 /// getSplatIdxForPPCMnemonics - Return the splat index as a value that is 2421 /// appropriate for PPC mnemonics (which have a big endian bias - namely 2422 /// elements are counted from the left of the vector register). 2423 unsigned PPC::getSplatIdxForPPCMnemonics(SDNode *N, unsigned EltSize, 2424 SelectionDAG &DAG) { 2425 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 2426 assert(isSplatShuffleMask(SVOp, EltSize)); 2427 if (DAG.getDataLayout().isLittleEndian()) 2428 return (16 / EltSize) - 1 - (SVOp->getMaskElt(0) / EltSize); 2429 else 2430 return SVOp->getMaskElt(0) / EltSize; 2431 } 2432 2433 /// get_VSPLTI_elt - If this is a build_vector of constants which can be formed 2434 /// by using a vspltis[bhw] instruction of the specified element size, return 2435 /// the constant being splatted. The ByteSize field indicates the number of 2436 /// bytes of each element [124] -> [bhw]. 2437 SDValue PPC::get_VSPLTI_elt(SDNode *N, unsigned ByteSize, SelectionDAG &DAG) { 2438 SDValue OpVal; 2439 2440 // If ByteSize of the splat is bigger than the element size of the 2441 // build_vector, then we have a case where we are checking for a splat where 2442 // multiple elements of the buildvector are folded together into a single 2443 // logical element of the splat (e.g. "vsplish 1" to splat {0,1}*8). 2444 unsigned EltSize = 16/N->getNumOperands(); 2445 if (EltSize < ByteSize) { 2446 unsigned Multiple = ByteSize/EltSize; // Number of BV entries per spltval. 2447 SDValue UniquedVals[4]; 2448 assert(Multiple > 1 && Multiple <= 4 && "How can this happen?"); 2449 2450 // See if all of the elements in the buildvector agree across. 2451 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 2452 if (N->getOperand(i).isUndef()) continue; 2453 // If the element isn't a constant, bail fully out. 2454 if (!isa<ConstantSDNode>(N->getOperand(i))) return SDValue(); 2455 2456 if (!UniquedVals[i&(Multiple-1)].getNode()) 2457 UniquedVals[i&(Multiple-1)] = N->getOperand(i); 2458 else if (UniquedVals[i&(Multiple-1)] != N->getOperand(i)) 2459 return SDValue(); // no match. 2460 } 2461 2462 // Okay, if we reached this point, UniquedVals[0..Multiple-1] contains 2463 // either constant or undef values that are identical for each chunk. See 2464 // if these chunks can form into a larger vspltis*. 2465 2466 // Check to see if all of the leading entries are either 0 or -1. If 2467 // neither, then this won't fit into the immediate field. 2468 bool LeadingZero = true; 2469 bool LeadingOnes = true; 2470 for (unsigned i = 0; i != Multiple-1; ++i) { 2471 if (!UniquedVals[i].getNode()) continue; // Must have been undefs. 2472 2473 LeadingZero &= isNullConstant(UniquedVals[i]); 2474 LeadingOnes &= isAllOnesConstant(UniquedVals[i]); 2475 } 2476 // Finally, check the least significant entry. 2477 if (LeadingZero) { 2478 if (!UniquedVals[Multiple-1].getNode()) 2479 return DAG.getTargetConstant(0, SDLoc(N), MVT::i32); // 0,0,0,undef 2480 int Val = cast<ConstantSDNode>(UniquedVals[Multiple-1])->getZExtValue(); 2481 if (Val < 16) // 0,0,0,4 -> vspltisw(4) 2482 return DAG.getTargetConstant(Val, SDLoc(N), MVT::i32); 2483 } 2484 if (LeadingOnes) { 2485 if (!UniquedVals[Multiple-1].getNode()) 2486 return DAG.getTargetConstant(~0U, SDLoc(N), MVT::i32); // -1,-1,-1,undef 2487 int Val =cast<ConstantSDNode>(UniquedVals[Multiple-1])->getSExtValue(); 2488 if (Val >= -16) // -1,-1,-1,-2 -> vspltisw(-2) 2489 return DAG.getTargetConstant(Val, SDLoc(N), MVT::i32); 2490 } 2491 2492 return SDValue(); 2493 } 2494 2495 // Check to see if this buildvec has a single non-undef value in its elements. 2496 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 2497 if (N->getOperand(i).isUndef()) continue; 2498 if (!OpVal.getNode()) 2499 OpVal = N->getOperand(i); 2500 else if (OpVal != N->getOperand(i)) 2501 return SDValue(); 2502 } 2503 2504 if (!OpVal.getNode()) return SDValue(); // All UNDEF: use implicit def. 2505 2506 unsigned ValSizeInBytes = EltSize; 2507 uint64_t Value = 0; 2508 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) { 2509 Value = CN->getZExtValue(); 2510 } else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal)) { 2511 assert(CN->getValueType(0) == MVT::f32 && "Only one legal FP vector type!"); 2512 Value = FloatToBits(CN->getValueAPF().convertToFloat()); 2513 } 2514 2515 // If the splat value is larger than the element value, then we can never do 2516 // this splat. The only case that we could fit the replicated bits into our 2517 // immediate field for would be zero, and we prefer to use vxor for it. 2518 if (ValSizeInBytes < ByteSize) return SDValue(); 2519 2520 // If the element value is larger than the splat value, check if it consists 2521 // of a repeated bit pattern of size ByteSize. 2522 if (!APInt(ValSizeInBytes * 8, Value).isSplat(ByteSize * 8)) 2523 return SDValue(); 2524 2525 // Properly sign extend the value. 2526 int MaskVal = SignExtend32(Value, ByteSize * 8); 2527 2528 // If this is zero, don't match, zero matches ISD::isBuildVectorAllZeros. 2529 if (MaskVal == 0) return SDValue(); 2530 2531 // Finally, if this value fits in a 5 bit sext field, return it 2532 if (SignExtend32<5>(MaskVal) == MaskVal) 2533 return DAG.getTargetConstant(MaskVal, SDLoc(N), MVT::i32); 2534 return SDValue(); 2535 } 2536 2537 //===----------------------------------------------------------------------===// 2538 // Addressing Mode Selection 2539 //===----------------------------------------------------------------------===// 2540 2541 /// isIntS16Immediate - This method tests to see if the node is either a 32-bit 2542 /// or 64-bit immediate, and if the value can be accurately represented as a 2543 /// sign extension from a 16-bit value. If so, this returns true and the 2544 /// immediate. 2545 bool llvm::isIntS16Immediate(SDNode *N, int16_t &Imm) { 2546 if (!isa<ConstantSDNode>(N)) 2547 return false; 2548 2549 Imm = (int16_t)cast<ConstantSDNode>(N)->getZExtValue(); 2550 if (N->getValueType(0) == MVT::i32) 2551 return Imm == (int32_t)cast<ConstantSDNode>(N)->getZExtValue(); 2552 else 2553 return Imm == (int64_t)cast<ConstantSDNode>(N)->getZExtValue(); 2554 } 2555 bool llvm::isIntS16Immediate(SDValue Op, int16_t &Imm) { 2556 return isIntS16Immediate(Op.getNode(), Imm); 2557 } 2558 2559 /// Used when computing address flags for selecting loads and stores. 2560 /// If we have an OR, check if the LHS and RHS are provably disjoint. 2561 /// An OR of two provably disjoint values is equivalent to an ADD. 2562 /// Most PPC load/store instructions compute the effective address as a sum, 2563 /// so doing this conversion is useful. 2564 static bool provablyDisjointOr(SelectionDAG &DAG, const SDValue &N) { 2565 if (N.getOpcode() != ISD::OR) 2566 return false; 2567 KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0)); 2568 if (!LHSKnown.Zero.getBoolValue()) 2569 return false; 2570 KnownBits RHSKnown = DAG.computeKnownBits(N.getOperand(1)); 2571 return (~(LHSKnown.Zero | RHSKnown.Zero) == 0); 2572 } 2573 2574 /// SelectAddressEVXRegReg - Given the specified address, check to see if it can 2575 /// be represented as an indexed [r+r] operation. 2576 bool PPCTargetLowering::SelectAddressEVXRegReg(SDValue N, SDValue &Base, 2577 SDValue &Index, 2578 SelectionDAG &DAG) const { 2579 for (SDNode *U : N->uses()) { 2580 if (MemSDNode *Memop = dyn_cast<MemSDNode>(U)) { 2581 if (Memop->getMemoryVT() == MVT::f64) { 2582 Base = N.getOperand(0); 2583 Index = N.getOperand(1); 2584 return true; 2585 } 2586 } 2587 } 2588 return false; 2589 } 2590 2591 /// isIntS34Immediate - This method tests if value of node given can be 2592 /// accurately represented as a sign extension from a 34-bit value. If so, 2593 /// this returns true and the immediate. 2594 bool llvm::isIntS34Immediate(SDNode *N, int64_t &Imm) { 2595 if (!isa<ConstantSDNode>(N)) 2596 return false; 2597 2598 Imm = (int64_t)cast<ConstantSDNode>(N)->getZExtValue(); 2599 return isInt<34>(Imm); 2600 } 2601 bool llvm::isIntS34Immediate(SDValue Op, int64_t &Imm) { 2602 return isIntS34Immediate(Op.getNode(), Imm); 2603 } 2604 2605 /// SelectAddressRegReg - Given the specified addressed, check to see if it 2606 /// can be represented as an indexed [r+r] operation. Returns false if it 2607 /// can be more efficiently represented as [r+imm]. If \p EncodingAlignment is 2608 /// non-zero and N can be represented by a base register plus a signed 16-bit 2609 /// displacement, make a more precise judgement by checking (displacement % \p 2610 /// EncodingAlignment). 2611 bool PPCTargetLowering::SelectAddressRegReg( 2612 SDValue N, SDValue &Base, SDValue &Index, SelectionDAG &DAG, 2613 MaybeAlign EncodingAlignment) const { 2614 // If we have a PC Relative target flag don't select as [reg+reg]. It will be 2615 // a [pc+imm]. 2616 if (SelectAddressPCRel(N, Base)) 2617 return false; 2618 2619 int16_t Imm = 0; 2620 if (N.getOpcode() == ISD::ADD) { 2621 // Is there any SPE load/store (f64), which can't handle 16bit offset? 2622 // SPE load/store can only handle 8-bit offsets. 2623 if (hasSPE() && SelectAddressEVXRegReg(N, Base, Index, DAG)) 2624 return true; 2625 if (isIntS16Immediate(N.getOperand(1), Imm) && 2626 (!EncodingAlignment || isAligned(*EncodingAlignment, Imm))) 2627 return false; // r+i 2628 if (N.getOperand(1).getOpcode() == PPCISD::Lo) 2629 return false; // r+i 2630 2631 Base = N.getOperand(0); 2632 Index = N.getOperand(1); 2633 return true; 2634 } else if (N.getOpcode() == ISD::OR) { 2635 if (isIntS16Immediate(N.getOperand(1), Imm) && 2636 (!EncodingAlignment || isAligned(*EncodingAlignment, Imm))) 2637 return false; // r+i can fold it if we can. 2638 2639 // If this is an or of disjoint bitfields, we can codegen this as an add 2640 // (for better address arithmetic) if the LHS and RHS of the OR are provably 2641 // disjoint. 2642 KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0)); 2643 2644 if (LHSKnown.Zero.getBoolValue()) { 2645 KnownBits RHSKnown = DAG.computeKnownBits(N.getOperand(1)); 2646 // If all of the bits are known zero on the LHS or RHS, the add won't 2647 // carry. 2648 if (~(LHSKnown.Zero | RHSKnown.Zero) == 0) { 2649 Base = N.getOperand(0); 2650 Index = N.getOperand(1); 2651 return true; 2652 } 2653 } 2654 } 2655 2656 return false; 2657 } 2658 2659 // If we happen to be doing an i64 load or store into a stack slot that has 2660 // less than a 4-byte alignment, then the frame-index elimination may need to 2661 // use an indexed load or store instruction (because the offset may not be a 2662 // multiple of 4). The extra register needed to hold the offset comes from the 2663 // register scavenger, and it is possible that the scavenger will need to use 2664 // an emergency spill slot. As a result, we need to make sure that a spill slot 2665 // is allocated when doing an i64 load/store into a less-than-4-byte-aligned 2666 // stack slot. 2667 static void fixupFuncForFI(SelectionDAG &DAG, int FrameIdx, EVT VT) { 2668 // FIXME: This does not handle the LWA case. 2669 if (VT != MVT::i64) 2670 return; 2671 2672 // NOTE: We'll exclude negative FIs here, which come from argument 2673 // lowering, because there are no known test cases triggering this problem 2674 // using packed structures (or similar). We can remove this exclusion if 2675 // we find such a test case. The reason why this is so test-case driven is 2676 // because this entire 'fixup' is only to prevent crashes (from the 2677 // register scavenger) on not-really-valid inputs. For example, if we have: 2678 // %a = alloca i1 2679 // %b = bitcast i1* %a to i64* 2680 // store i64* a, i64 b 2681 // then the store should really be marked as 'align 1', but is not. If it 2682 // were marked as 'align 1' then the indexed form would have been 2683 // instruction-selected initially, and the problem this 'fixup' is preventing 2684 // won't happen regardless. 2685 if (FrameIdx < 0) 2686 return; 2687 2688 MachineFunction &MF = DAG.getMachineFunction(); 2689 MachineFrameInfo &MFI = MF.getFrameInfo(); 2690 2691 if (MFI.getObjectAlign(FrameIdx) >= Align(4)) 2692 return; 2693 2694 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 2695 FuncInfo->setHasNonRISpills(); 2696 } 2697 2698 /// Returns true if the address N can be represented by a base register plus 2699 /// a signed 16-bit displacement [r+imm], and if it is not better 2700 /// represented as reg+reg. If \p EncodingAlignment is non-zero, only accept 2701 /// displacements that are multiples of that value. 2702 bool PPCTargetLowering::SelectAddressRegImm( 2703 SDValue N, SDValue &Disp, SDValue &Base, SelectionDAG &DAG, 2704 MaybeAlign EncodingAlignment) const { 2705 // FIXME dl should come from parent load or store, not from address 2706 SDLoc dl(N); 2707 2708 // If we have a PC Relative target flag don't select as [reg+imm]. It will be 2709 // a [pc+imm]. 2710 if (SelectAddressPCRel(N, Base)) 2711 return false; 2712 2713 // If this can be more profitably realized as r+r, fail. 2714 if (SelectAddressRegReg(N, Disp, Base, DAG, EncodingAlignment)) 2715 return false; 2716 2717 if (N.getOpcode() == ISD::ADD) { 2718 int16_t imm = 0; 2719 if (isIntS16Immediate(N.getOperand(1), imm) && 2720 (!EncodingAlignment || isAligned(*EncodingAlignment, imm))) { 2721 Disp = DAG.getTargetConstant(imm, dl, N.getValueType()); 2722 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N.getOperand(0))) { 2723 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); 2724 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); 2725 } else { 2726 Base = N.getOperand(0); 2727 } 2728 return true; // [r+i] 2729 } else if (N.getOperand(1).getOpcode() == PPCISD::Lo) { 2730 // Match LOAD (ADD (X, Lo(G))). 2731 assert(!cast<ConstantSDNode>(N.getOperand(1).getOperand(1))->getZExtValue() 2732 && "Cannot handle constant offsets yet!"); 2733 Disp = N.getOperand(1).getOperand(0); // The global address. 2734 assert(Disp.getOpcode() == ISD::TargetGlobalAddress || 2735 Disp.getOpcode() == ISD::TargetGlobalTLSAddress || 2736 Disp.getOpcode() == ISD::TargetConstantPool || 2737 Disp.getOpcode() == ISD::TargetJumpTable); 2738 Base = N.getOperand(0); 2739 return true; // [&g+r] 2740 } 2741 } else if (N.getOpcode() == ISD::OR) { 2742 int16_t imm = 0; 2743 if (isIntS16Immediate(N.getOperand(1), imm) && 2744 (!EncodingAlignment || isAligned(*EncodingAlignment, imm))) { 2745 // If this is an or of disjoint bitfields, we can codegen this as an add 2746 // (for better address arithmetic) if the LHS and RHS of the OR are 2747 // provably disjoint. 2748 KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0)); 2749 2750 if ((LHSKnown.Zero.getZExtValue()|~(uint64_t)imm) == ~0ULL) { 2751 // If all of the bits are known zero on the LHS or RHS, the add won't 2752 // carry. 2753 if (FrameIndexSDNode *FI = 2754 dyn_cast<FrameIndexSDNode>(N.getOperand(0))) { 2755 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); 2756 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); 2757 } else { 2758 Base = N.getOperand(0); 2759 } 2760 Disp = DAG.getTargetConstant(imm, dl, N.getValueType()); 2761 return true; 2762 } 2763 } 2764 } else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) { 2765 // Loading from a constant address. 2766 2767 // If this address fits entirely in a 16-bit sext immediate field, codegen 2768 // this as "d, 0" 2769 int16_t Imm; 2770 if (isIntS16Immediate(CN, Imm) && 2771 (!EncodingAlignment || isAligned(*EncodingAlignment, Imm))) { 2772 Disp = DAG.getTargetConstant(Imm, dl, CN->getValueType(0)); 2773 Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO, 2774 CN->getValueType(0)); 2775 return true; 2776 } 2777 2778 // Handle 32-bit sext immediates with LIS + addr mode. 2779 if ((CN->getValueType(0) == MVT::i32 || 2780 (int64_t)CN->getZExtValue() == (int)CN->getZExtValue()) && 2781 (!EncodingAlignment || 2782 isAligned(*EncodingAlignment, CN->getZExtValue()))) { 2783 int Addr = (int)CN->getZExtValue(); 2784 2785 // Otherwise, break this down into an LIS + disp. 2786 Disp = DAG.getTargetConstant((short)Addr, dl, MVT::i32); 2787 2788 Base = DAG.getTargetConstant((Addr - (signed short)Addr) >> 16, dl, 2789 MVT::i32); 2790 unsigned Opc = CN->getValueType(0) == MVT::i32 ? PPC::LIS : PPC::LIS8; 2791 Base = SDValue(DAG.getMachineNode(Opc, dl, CN->getValueType(0), Base), 0); 2792 return true; 2793 } 2794 } 2795 2796 Disp = DAG.getTargetConstant(0, dl, getPointerTy(DAG.getDataLayout())); 2797 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N)) { 2798 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); 2799 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); 2800 } else 2801 Base = N; 2802 return true; // [r+0] 2803 } 2804 2805 /// Similar to the 16-bit case but for instructions that take a 34-bit 2806 /// displacement field (prefixed loads/stores). 2807 bool PPCTargetLowering::SelectAddressRegImm34(SDValue N, SDValue &Disp, 2808 SDValue &Base, 2809 SelectionDAG &DAG) const { 2810 // Only on 64-bit targets. 2811 if (N.getValueType() != MVT::i64) 2812 return false; 2813 2814 SDLoc dl(N); 2815 int64_t Imm = 0; 2816 2817 if (N.getOpcode() == ISD::ADD) { 2818 if (!isIntS34Immediate(N.getOperand(1), Imm)) 2819 return false; 2820 Disp = DAG.getTargetConstant(Imm, dl, N.getValueType()); 2821 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N.getOperand(0))) 2822 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); 2823 else 2824 Base = N.getOperand(0); 2825 return true; 2826 } 2827 2828 if (N.getOpcode() == ISD::OR) { 2829 if (!isIntS34Immediate(N.getOperand(1), Imm)) 2830 return false; 2831 // If this is an or of disjoint bitfields, we can codegen this as an add 2832 // (for better address arithmetic) if the LHS and RHS of the OR are 2833 // provably disjoint. 2834 KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0)); 2835 if ((LHSKnown.Zero.getZExtValue() | ~(uint64_t)Imm) != ~0ULL) 2836 return false; 2837 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N.getOperand(0))) 2838 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); 2839 else 2840 Base = N.getOperand(0); 2841 Disp = DAG.getTargetConstant(Imm, dl, N.getValueType()); 2842 return true; 2843 } 2844 2845 if (isIntS34Immediate(N, Imm)) { // If the address is a 34-bit const. 2846 Disp = DAG.getTargetConstant(Imm, dl, N.getValueType()); 2847 Base = DAG.getRegister(PPC::ZERO8, N.getValueType()); 2848 return true; 2849 } 2850 2851 return false; 2852 } 2853 2854 /// SelectAddressRegRegOnly - Given the specified addressed, force it to be 2855 /// represented as an indexed [r+r] operation. 2856 bool PPCTargetLowering::SelectAddressRegRegOnly(SDValue N, SDValue &Base, 2857 SDValue &Index, 2858 SelectionDAG &DAG) const { 2859 // Check to see if we can easily represent this as an [r+r] address. This 2860 // will fail if it thinks that the address is more profitably represented as 2861 // reg+imm, e.g. where imm = 0. 2862 if (SelectAddressRegReg(N, Base, Index, DAG)) 2863 return true; 2864 2865 // If the address is the result of an add, we will utilize the fact that the 2866 // address calculation includes an implicit add. However, we can reduce 2867 // register pressure if we do not materialize a constant just for use as the 2868 // index register. We only get rid of the add if it is not an add of a 2869 // value and a 16-bit signed constant and both have a single use. 2870 int16_t imm = 0; 2871 if (N.getOpcode() == ISD::ADD && 2872 (!isIntS16Immediate(N.getOperand(1), imm) || 2873 !N.getOperand(1).hasOneUse() || !N.getOperand(0).hasOneUse())) { 2874 Base = N.getOperand(0); 2875 Index = N.getOperand(1); 2876 return true; 2877 } 2878 2879 // Otherwise, do it the hard way, using R0 as the base register. 2880 Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO, 2881 N.getValueType()); 2882 Index = N; 2883 return true; 2884 } 2885 2886 template <typename Ty> static bool isValidPCRelNode(SDValue N) { 2887 Ty *PCRelCand = dyn_cast<Ty>(N); 2888 return PCRelCand && (PCRelCand->getTargetFlags() & PPCII::MO_PCREL_FLAG); 2889 } 2890 2891 /// Returns true if this address is a PC Relative address. 2892 /// PC Relative addresses are marked with the flag PPCII::MO_PCREL_FLAG 2893 /// or if the node opcode is PPCISD::MAT_PCREL_ADDR. 2894 bool PPCTargetLowering::SelectAddressPCRel(SDValue N, SDValue &Base) const { 2895 // This is a materialize PC Relative node. Always select this as PC Relative. 2896 Base = N; 2897 if (N.getOpcode() == PPCISD::MAT_PCREL_ADDR) 2898 return true; 2899 if (isValidPCRelNode<ConstantPoolSDNode>(N) || 2900 isValidPCRelNode<GlobalAddressSDNode>(N) || 2901 isValidPCRelNode<JumpTableSDNode>(N) || 2902 isValidPCRelNode<BlockAddressSDNode>(N)) 2903 return true; 2904 return false; 2905 } 2906 2907 /// Returns true if we should use a direct load into vector instruction 2908 /// (such as lxsd or lfd), instead of a load into gpr + direct move sequence. 2909 static bool usePartialVectorLoads(SDNode *N, const PPCSubtarget& ST) { 2910 2911 // If there are any other uses other than scalar to vector, then we should 2912 // keep it as a scalar load -> direct move pattern to prevent multiple 2913 // loads. 2914 LoadSDNode *LD = dyn_cast<LoadSDNode>(N); 2915 if (!LD) 2916 return false; 2917 2918 EVT MemVT = LD->getMemoryVT(); 2919 if (!MemVT.isSimple()) 2920 return false; 2921 switch(MemVT.getSimpleVT().SimpleTy) { 2922 case MVT::i64: 2923 break; 2924 case MVT::i32: 2925 if (!ST.hasP8Vector()) 2926 return false; 2927 break; 2928 case MVT::i16: 2929 case MVT::i8: 2930 if (!ST.hasP9Vector()) 2931 return false; 2932 break; 2933 default: 2934 return false; 2935 } 2936 2937 SDValue LoadedVal(N, 0); 2938 if (!LoadedVal.hasOneUse()) 2939 return false; 2940 2941 for (SDNode::use_iterator UI = LD->use_begin(), UE = LD->use_end(); 2942 UI != UE; ++UI) 2943 if (UI.getUse().get().getResNo() == 0 && 2944 UI->getOpcode() != ISD::SCALAR_TO_VECTOR && 2945 UI->getOpcode() != PPCISD::SCALAR_TO_VECTOR_PERMUTED) 2946 return false; 2947 2948 return true; 2949 } 2950 2951 /// getPreIndexedAddressParts - returns true by value, base pointer and 2952 /// offset pointer and addressing mode by reference if the node's address 2953 /// can be legally represented as pre-indexed load / store address. 2954 bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base, 2955 SDValue &Offset, 2956 ISD::MemIndexedMode &AM, 2957 SelectionDAG &DAG) const { 2958 if (DisablePPCPreinc) return false; 2959 2960 bool isLoad = true; 2961 SDValue Ptr; 2962 EVT VT; 2963 unsigned Alignment; 2964 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) { 2965 Ptr = LD->getBasePtr(); 2966 VT = LD->getMemoryVT(); 2967 Alignment = LD->getAlignment(); 2968 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) { 2969 Ptr = ST->getBasePtr(); 2970 VT = ST->getMemoryVT(); 2971 Alignment = ST->getAlignment(); 2972 isLoad = false; 2973 } else 2974 return false; 2975 2976 // Do not generate pre-inc forms for specific loads that feed scalar_to_vector 2977 // instructions because we can fold these into a more efficient instruction 2978 // instead, (such as LXSD). 2979 if (isLoad && usePartialVectorLoads(N, Subtarget)) { 2980 return false; 2981 } 2982 2983 // PowerPC doesn't have preinc load/store instructions for vectors 2984 if (VT.isVector()) 2985 return false; 2986 2987 if (SelectAddressRegReg(Ptr, Base, Offset, DAG)) { 2988 // Common code will reject creating a pre-inc form if the base pointer 2989 // is a frame index, or if N is a store and the base pointer is either 2990 // the same as or a predecessor of the value being stored. Check for 2991 // those situations here, and try with swapped Base/Offset instead. 2992 bool Swap = false; 2993 2994 if (isa<FrameIndexSDNode>(Base) || isa<RegisterSDNode>(Base)) 2995 Swap = true; 2996 else if (!isLoad) { 2997 SDValue Val = cast<StoreSDNode>(N)->getValue(); 2998 if (Val == Base || Base.getNode()->isPredecessorOf(Val.getNode())) 2999 Swap = true; 3000 } 3001 3002 if (Swap) 3003 std::swap(Base, Offset); 3004 3005 AM = ISD::PRE_INC; 3006 return true; 3007 } 3008 3009 // LDU/STU can only handle immediates that are a multiple of 4. 3010 if (VT != MVT::i64) { 3011 if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, None)) 3012 return false; 3013 } else { 3014 // LDU/STU need an address with at least 4-byte alignment. 3015 if (Alignment < 4) 3016 return false; 3017 3018 if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, Align(4))) 3019 return false; 3020 } 3021 3022 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) { 3023 // PPC64 doesn't have lwau, but it does have lwaux. Reject preinc load of 3024 // sext i32 to i64 when addr mode is r+i. 3025 if (LD->getValueType(0) == MVT::i64 && LD->getMemoryVT() == MVT::i32 && 3026 LD->getExtensionType() == ISD::SEXTLOAD && 3027 isa<ConstantSDNode>(Offset)) 3028 return false; 3029 } 3030 3031 AM = ISD::PRE_INC; 3032 return true; 3033 } 3034 3035 //===----------------------------------------------------------------------===// 3036 // LowerOperation implementation 3037 //===----------------------------------------------------------------------===// 3038 3039 /// Return true if we should reference labels using a PICBase, set the HiOpFlags 3040 /// and LoOpFlags to the target MO flags. 3041 static void getLabelAccessInfo(bool IsPIC, const PPCSubtarget &Subtarget, 3042 unsigned &HiOpFlags, unsigned &LoOpFlags, 3043 const GlobalValue *GV = nullptr) { 3044 HiOpFlags = PPCII::MO_HA; 3045 LoOpFlags = PPCII::MO_LO; 3046 3047 // Don't use the pic base if not in PIC relocation model. 3048 if (IsPIC) { 3049 HiOpFlags |= PPCII::MO_PIC_FLAG; 3050 LoOpFlags |= PPCII::MO_PIC_FLAG; 3051 } 3052 } 3053 3054 static SDValue LowerLabelRef(SDValue HiPart, SDValue LoPart, bool isPIC, 3055 SelectionDAG &DAG) { 3056 SDLoc DL(HiPart); 3057 EVT PtrVT = HiPart.getValueType(); 3058 SDValue Zero = DAG.getConstant(0, DL, PtrVT); 3059 3060 SDValue Hi = DAG.getNode(PPCISD::Hi, DL, PtrVT, HiPart, Zero); 3061 SDValue Lo = DAG.getNode(PPCISD::Lo, DL, PtrVT, LoPart, Zero); 3062 3063 // With PIC, the first instruction is actually "GR+hi(&G)". 3064 if (isPIC) 3065 Hi = DAG.getNode(ISD::ADD, DL, PtrVT, 3066 DAG.getNode(PPCISD::GlobalBaseReg, DL, PtrVT), Hi); 3067 3068 // Generate non-pic code that has direct accesses to the constant pool. 3069 // The address of the global is just (hi(&g)+lo(&g)). 3070 return DAG.getNode(ISD::ADD, DL, PtrVT, Hi, Lo); 3071 } 3072 3073 static void setUsesTOCBasePtr(MachineFunction &MF) { 3074 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 3075 FuncInfo->setUsesTOCBasePtr(); 3076 } 3077 3078 static void setUsesTOCBasePtr(SelectionDAG &DAG) { 3079 setUsesTOCBasePtr(DAG.getMachineFunction()); 3080 } 3081 3082 SDValue PPCTargetLowering::getTOCEntry(SelectionDAG &DAG, const SDLoc &dl, 3083 SDValue GA) const { 3084 const bool Is64Bit = Subtarget.isPPC64(); 3085 EVT VT = Is64Bit ? MVT::i64 : MVT::i32; 3086 SDValue Reg = Is64Bit ? DAG.getRegister(PPC::X2, VT) 3087 : Subtarget.isAIXABI() 3088 ? DAG.getRegister(PPC::R2, VT) 3089 : DAG.getNode(PPCISD::GlobalBaseReg, dl, VT); 3090 SDValue Ops[] = { GA, Reg }; 3091 return DAG.getMemIntrinsicNode( 3092 PPCISD::TOC_ENTRY, dl, DAG.getVTList(VT, MVT::Other), Ops, VT, 3093 MachinePointerInfo::getGOT(DAG.getMachineFunction()), None, 3094 MachineMemOperand::MOLoad); 3095 } 3096 3097 SDValue PPCTargetLowering::LowerConstantPool(SDValue Op, 3098 SelectionDAG &DAG) const { 3099 EVT PtrVT = Op.getValueType(); 3100 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op); 3101 const Constant *C = CP->getConstVal(); 3102 3103 // 64-bit SVR4 ABI and AIX ABI code are always position-independent. 3104 // The actual address of the GlobalValue is stored in the TOC. 3105 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) { 3106 if (Subtarget.isUsingPCRelativeCalls()) { 3107 SDLoc DL(CP); 3108 EVT Ty = getPointerTy(DAG.getDataLayout()); 3109 SDValue ConstPool = DAG.getTargetConstantPool( 3110 C, Ty, CP->getAlign(), CP->getOffset(), PPCII::MO_PCREL_FLAG); 3111 return DAG.getNode(PPCISD::MAT_PCREL_ADDR, DL, Ty, ConstPool); 3112 } 3113 setUsesTOCBasePtr(DAG); 3114 SDValue GA = DAG.getTargetConstantPool(C, PtrVT, CP->getAlign(), 0); 3115 return getTOCEntry(DAG, SDLoc(CP), GA); 3116 } 3117 3118 unsigned MOHiFlag, MOLoFlag; 3119 bool IsPIC = isPositionIndependent(); 3120 getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag); 3121 3122 if (IsPIC && Subtarget.isSVR4ABI()) { 3123 SDValue GA = 3124 DAG.getTargetConstantPool(C, PtrVT, CP->getAlign(), PPCII::MO_PIC_FLAG); 3125 return getTOCEntry(DAG, SDLoc(CP), GA); 3126 } 3127 3128 SDValue CPIHi = 3129 DAG.getTargetConstantPool(C, PtrVT, CP->getAlign(), 0, MOHiFlag); 3130 SDValue CPILo = 3131 DAG.getTargetConstantPool(C, PtrVT, CP->getAlign(), 0, MOLoFlag); 3132 return LowerLabelRef(CPIHi, CPILo, IsPIC, DAG); 3133 } 3134 3135 // For 64-bit PowerPC, prefer the more compact relative encodings. 3136 // This trades 32 bits per jump table entry for one or two instructions 3137 // on the jump site. 3138 unsigned PPCTargetLowering::getJumpTableEncoding() const { 3139 if (isJumpTableRelative()) 3140 return MachineJumpTableInfo::EK_LabelDifference32; 3141 3142 return TargetLowering::getJumpTableEncoding(); 3143 } 3144 3145 bool PPCTargetLowering::isJumpTableRelative() const { 3146 if (UseAbsoluteJumpTables) 3147 return false; 3148 if (Subtarget.isPPC64() || Subtarget.isAIXABI()) 3149 return true; 3150 return TargetLowering::isJumpTableRelative(); 3151 } 3152 3153 SDValue PPCTargetLowering::getPICJumpTableRelocBase(SDValue Table, 3154 SelectionDAG &DAG) const { 3155 if (!Subtarget.isPPC64() || Subtarget.isAIXABI()) 3156 return TargetLowering::getPICJumpTableRelocBase(Table, DAG); 3157 3158 switch (getTargetMachine().getCodeModel()) { 3159 case CodeModel::Small: 3160 case CodeModel::Medium: 3161 return TargetLowering::getPICJumpTableRelocBase(Table, DAG); 3162 default: 3163 return DAG.getNode(PPCISD::GlobalBaseReg, SDLoc(), 3164 getPointerTy(DAG.getDataLayout())); 3165 } 3166 } 3167 3168 const MCExpr * 3169 PPCTargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF, 3170 unsigned JTI, 3171 MCContext &Ctx) const { 3172 if (!Subtarget.isPPC64() || Subtarget.isAIXABI()) 3173 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx); 3174 3175 switch (getTargetMachine().getCodeModel()) { 3176 case CodeModel::Small: 3177 case CodeModel::Medium: 3178 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx); 3179 default: 3180 return MCSymbolRefExpr::create(MF->getPICBaseSymbol(), Ctx); 3181 } 3182 } 3183 3184 SDValue PPCTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const { 3185 EVT PtrVT = Op.getValueType(); 3186 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op); 3187 3188 // isUsingPCRelativeCalls() returns true when PCRelative is enabled 3189 if (Subtarget.isUsingPCRelativeCalls()) { 3190 SDLoc DL(JT); 3191 EVT Ty = getPointerTy(DAG.getDataLayout()); 3192 SDValue GA = 3193 DAG.getTargetJumpTable(JT->getIndex(), Ty, PPCII::MO_PCREL_FLAG); 3194 SDValue MatAddr = DAG.getNode(PPCISD::MAT_PCREL_ADDR, DL, Ty, GA); 3195 return MatAddr; 3196 } 3197 3198 // 64-bit SVR4 ABI and AIX ABI code are always position-independent. 3199 // The actual address of the GlobalValue is stored in the TOC. 3200 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) { 3201 setUsesTOCBasePtr(DAG); 3202 SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT); 3203 return getTOCEntry(DAG, SDLoc(JT), GA); 3204 } 3205 3206 unsigned MOHiFlag, MOLoFlag; 3207 bool IsPIC = isPositionIndependent(); 3208 getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag); 3209 3210 if (IsPIC && Subtarget.isSVR4ABI()) { 3211 SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, 3212 PPCII::MO_PIC_FLAG); 3213 return getTOCEntry(DAG, SDLoc(GA), GA); 3214 } 3215 3216 SDValue JTIHi = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOHiFlag); 3217 SDValue JTILo = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOLoFlag); 3218 return LowerLabelRef(JTIHi, JTILo, IsPIC, DAG); 3219 } 3220 3221 SDValue PPCTargetLowering::LowerBlockAddress(SDValue Op, 3222 SelectionDAG &DAG) const { 3223 EVT PtrVT = Op.getValueType(); 3224 BlockAddressSDNode *BASDN = cast<BlockAddressSDNode>(Op); 3225 const BlockAddress *BA = BASDN->getBlockAddress(); 3226 3227 // isUsingPCRelativeCalls() returns true when PCRelative is enabled 3228 if (Subtarget.isUsingPCRelativeCalls()) { 3229 SDLoc DL(BASDN); 3230 EVT Ty = getPointerTy(DAG.getDataLayout()); 3231 SDValue GA = DAG.getTargetBlockAddress(BA, Ty, BASDN->getOffset(), 3232 PPCII::MO_PCREL_FLAG); 3233 SDValue MatAddr = DAG.getNode(PPCISD::MAT_PCREL_ADDR, DL, Ty, GA); 3234 return MatAddr; 3235 } 3236 3237 // 64-bit SVR4 ABI and AIX ABI code are always position-independent. 3238 // The actual BlockAddress is stored in the TOC. 3239 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) { 3240 setUsesTOCBasePtr(DAG); 3241 SDValue GA = DAG.getTargetBlockAddress(BA, PtrVT, BASDN->getOffset()); 3242 return getTOCEntry(DAG, SDLoc(BASDN), GA); 3243 } 3244 3245 // 32-bit position-independent ELF stores the BlockAddress in the .got. 3246 if (Subtarget.is32BitELFABI() && isPositionIndependent()) 3247 return getTOCEntry( 3248 DAG, SDLoc(BASDN), 3249 DAG.getTargetBlockAddress(BA, PtrVT, BASDN->getOffset())); 3250 3251 unsigned MOHiFlag, MOLoFlag; 3252 bool IsPIC = isPositionIndependent(); 3253 getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag); 3254 SDValue TgtBAHi = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOHiFlag); 3255 SDValue TgtBALo = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOLoFlag); 3256 return LowerLabelRef(TgtBAHi, TgtBALo, IsPIC, DAG); 3257 } 3258 3259 SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op, 3260 SelectionDAG &DAG) const { 3261 if (Subtarget.isAIXABI()) 3262 return LowerGlobalTLSAddressAIX(Op, DAG); 3263 3264 return LowerGlobalTLSAddressLinux(Op, DAG); 3265 } 3266 3267 SDValue PPCTargetLowering::LowerGlobalTLSAddressAIX(SDValue Op, 3268 SelectionDAG &DAG) const { 3269 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); 3270 3271 if (DAG.getTarget().useEmulatedTLS()) 3272 report_fatal_error("Emulated TLS is not yet supported on AIX"); 3273 3274 SDLoc dl(GA); 3275 const GlobalValue *GV = GA->getGlobal(); 3276 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 3277 3278 // The general-dynamic model is the only access model supported for now, so 3279 // all the GlobalTLSAddress nodes are lowered with this model. 3280 // We need to generate two TOC entries, one for the variable offset, one for 3281 // the region handle. The global address for the TOC entry of the region 3282 // handle is created with the MO_TLSGDM_FLAG flag and the global address 3283 // for the TOC entry of the variable offset is created with MO_TLSGD_FLAG. 3284 SDValue VariableOffsetTGA = 3285 DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, PPCII::MO_TLSGD_FLAG); 3286 SDValue RegionHandleTGA = 3287 DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, PPCII::MO_TLSGDM_FLAG); 3288 SDValue VariableOffset = getTOCEntry(DAG, dl, VariableOffsetTGA); 3289 SDValue RegionHandle = getTOCEntry(DAG, dl, RegionHandleTGA); 3290 return DAG.getNode(PPCISD::TLSGD_AIX, dl, PtrVT, VariableOffset, 3291 RegionHandle); 3292 } 3293 3294 SDValue PPCTargetLowering::LowerGlobalTLSAddressLinux(SDValue Op, 3295 SelectionDAG &DAG) const { 3296 // FIXME: TLS addresses currently use medium model code sequences, 3297 // which is the most useful form. Eventually support for small and 3298 // large models could be added if users need it, at the cost of 3299 // additional complexity. 3300 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); 3301 if (DAG.getTarget().useEmulatedTLS()) 3302 return LowerToTLSEmulatedModel(GA, DAG); 3303 3304 SDLoc dl(GA); 3305 const GlobalValue *GV = GA->getGlobal(); 3306 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 3307 bool is64bit = Subtarget.isPPC64(); 3308 const Module *M = DAG.getMachineFunction().getFunction().getParent(); 3309 PICLevel::Level picLevel = M->getPICLevel(); 3310 3311 const TargetMachine &TM = getTargetMachine(); 3312 TLSModel::Model Model = TM.getTLSModel(GV); 3313 3314 if (Model == TLSModel::LocalExec) { 3315 if (Subtarget.isUsingPCRelativeCalls()) { 3316 SDValue TLSReg = DAG.getRegister(PPC::X13, MVT::i64); 3317 SDValue TGA = DAG.getTargetGlobalAddress( 3318 GV, dl, PtrVT, 0, (PPCII::MO_PCREL_FLAG | PPCII::MO_TPREL_FLAG)); 3319 SDValue MatAddr = 3320 DAG.getNode(PPCISD::TLS_LOCAL_EXEC_MAT_ADDR, dl, PtrVT, TGA); 3321 return DAG.getNode(PPCISD::ADD_TLS, dl, PtrVT, TLSReg, MatAddr); 3322 } 3323 3324 SDValue TGAHi = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 3325 PPCII::MO_TPREL_HA); 3326 SDValue TGALo = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 3327 PPCII::MO_TPREL_LO); 3328 SDValue TLSReg = is64bit ? DAG.getRegister(PPC::X13, MVT::i64) 3329 : DAG.getRegister(PPC::R2, MVT::i32); 3330 3331 SDValue Hi = DAG.getNode(PPCISD::Hi, dl, PtrVT, TGAHi, TLSReg); 3332 return DAG.getNode(PPCISD::Lo, dl, PtrVT, TGALo, Hi); 3333 } 3334 3335 if (Model == TLSModel::InitialExec) { 3336 bool IsPCRel = Subtarget.isUsingPCRelativeCalls(); 3337 SDValue TGA = DAG.getTargetGlobalAddress( 3338 GV, dl, PtrVT, 0, IsPCRel ? PPCII::MO_GOT_TPREL_PCREL_FLAG : 0); 3339 SDValue TGATLS = DAG.getTargetGlobalAddress( 3340 GV, dl, PtrVT, 0, 3341 IsPCRel ? (PPCII::MO_TLS | PPCII::MO_PCREL_FLAG) : PPCII::MO_TLS); 3342 SDValue TPOffset; 3343 if (IsPCRel) { 3344 SDValue MatPCRel = DAG.getNode(PPCISD::MAT_PCREL_ADDR, dl, PtrVT, TGA); 3345 TPOffset = DAG.getLoad(MVT::i64, dl, DAG.getEntryNode(), MatPCRel, 3346 MachinePointerInfo()); 3347 } else { 3348 SDValue GOTPtr; 3349 if (is64bit) { 3350 setUsesTOCBasePtr(DAG); 3351 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); 3352 GOTPtr = 3353 DAG.getNode(PPCISD::ADDIS_GOT_TPREL_HA, dl, PtrVT, GOTReg, TGA); 3354 } else { 3355 if (!TM.isPositionIndependent()) 3356 GOTPtr = DAG.getNode(PPCISD::PPC32_GOT, dl, PtrVT); 3357 else if (picLevel == PICLevel::SmallPIC) 3358 GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT); 3359 else 3360 GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT); 3361 } 3362 TPOffset = DAG.getNode(PPCISD::LD_GOT_TPREL_L, dl, PtrVT, TGA, GOTPtr); 3363 } 3364 return DAG.getNode(PPCISD::ADD_TLS, dl, PtrVT, TPOffset, TGATLS); 3365 } 3366 3367 if (Model == TLSModel::GeneralDynamic) { 3368 if (Subtarget.isUsingPCRelativeCalls()) { 3369 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 3370 PPCII::MO_GOT_TLSGD_PCREL_FLAG); 3371 return DAG.getNode(PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR, dl, PtrVT, TGA); 3372 } 3373 3374 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); 3375 SDValue GOTPtr; 3376 if (is64bit) { 3377 setUsesTOCBasePtr(DAG); 3378 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); 3379 GOTPtr = DAG.getNode(PPCISD::ADDIS_TLSGD_HA, dl, PtrVT, 3380 GOTReg, TGA); 3381 } else { 3382 if (picLevel == PICLevel::SmallPIC) 3383 GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT); 3384 else 3385 GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT); 3386 } 3387 return DAG.getNode(PPCISD::ADDI_TLSGD_L_ADDR, dl, PtrVT, 3388 GOTPtr, TGA, TGA); 3389 } 3390 3391 if (Model == TLSModel::LocalDynamic) { 3392 if (Subtarget.isUsingPCRelativeCalls()) { 3393 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 3394 PPCII::MO_GOT_TLSLD_PCREL_FLAG); 3395 SDValue MatPCRel = 3396 DAG.getNode(PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR, dl, PtrVT, TGA); 3397 return DAG.getNode(PPCISD::PADDI_DTPREL, dl, PtrVT, MatPCRel, TGA); 3398 } 3399 3400 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); 3401 SDValue GOTPtr; 3402 if (is64bit) { 3403 setUsesTOCBasePtr(DAG); 3404 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); 3405 GOTPtr = DAG.getNode(PPCISD::ADDIS_TLSLD_HA, dl, PtrVT, 3406 GOTReg, TGA); 3407 } else { 3408 if (picLevel == PICLevel::SmallPIC) 3409 GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT); 3410 else 3411 GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT); 3412 } 3413 SDValue TLSAddr = DAG.getNode(PPCISD::ADDI_TLSLD_L_ADDR, dl, 3414 PtrVT, GOTPtr, TGA, TGA); 3415 SDValue DtvOffsetHi = DAG.getNode(PPCISD::ADDIS_DTPREL_HA, dl, 3416 PtrVT, TLSAddr, TGA); 3417 return DAG.getNode(PPCISD::ADDI_DTPREL_L, dl, PtrVT, DtvOffsetHi, TGA); 3418 } 3419 3420 llvm_unreachable("Unknown TLS model!"); 3421 } 3422 3423 SDValue PPCTargetLowering::LowerGlobalAddress(SDValue Op, 3424 SelectionDAG &DAG) const { 3425 EVT PtrVT = Op.getValueType(); 3426 GlobalAddressSDNode *GSDN = cast<GlobalAddressSDNode>(Op); 3427 SDLoc DL(GSDN); 3428 const GlobalValue *GV = GSDN->getGlobal(); 3429 3430 // 64-bit SVR4 ABI & AIX ABI code is always position-independent. 3431 // The actual address of the GlobalValue is stored in the TOC. 3432 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) { 3433 if (Subtarget.isUsingPCRelativeCalls()) { 3434 EVT Ty = getPointerTy(DAG.getDataLayout()); 3435 if (isAccessedAsGotIndirect(Op)) { 3436 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, Ty, GSDN->getOffset(), 3437 PPCII::MO_PCREL_FLAG | 3438 PPCII::MO_GOT_FLAG); 3439 SDValue MatPCRel = DAG.getNode(PPCISD::MAT_PCREL_ADDR, DL, Ty, GA); 3440 SDValue Load = DAG.getLoad(MVT::i64, DL, DAG.getEntryNode(), MatPCRel, 3441 MachinePointerInfo()); 3442 return Load; 3443 } else { 3444 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, Ty, GSDN->getOffset(), 3445 PPCII::MO_PCREL_FLAG); 3446 return DAG.getNode(PPCISD::MAT_PCREL_ADDR, DL, Ty, GA); 3447 } 3448 } 3449 setUsesTOCBasePtr(DAG); 3450 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset()); 3451 return getTOCEntry(DAG, DL, GA); 3452 } 3453 3454 unsigned MOHiFlag, MOLoFlag; 3455 bool IsPIC = isPositionIndependent(); 3456 getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag, GV); 3457 3458 if (IsPIC && Subtarget.isSVR4ABI()) { 3459 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 3460 GSDN->getOffset(), 3461 PPCII::MO_PIC_FLAG); 3462 return getTOCEntry(DAG, DL, GA); 3463 } 3464 3465 SDValue GAHi = 3466 DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), MOHiFlag); 3467 SDValue GALo = 3468 DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), MOLoFlag); 3469 3470 return LowerLabelRef(GAHi, GALo, IsPIC, DAG); 3471 } 3472 3473 SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { 3474 bool IsStrict = Op->isStrictFPOpcode(); 3475 ISD::CondCode CC = 3476 cast<CondCodeSDNode>(Op.getOperand(IsStrict ? 3 : 2))->get(); 3477 SDValue LHS = Op.getOperand(IsStrict ? 1 : 0); 3478 SDValue RHS = Op.getOperand(IsStrict ? 2 : 1); 3479 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue(); 3480 EVT LHSVT = LHS.getValueType(); 3481 SDLoc dl(Op); 3482 3483 // Soften the setcc with libcall if it is fp128. 3484 if (LHSVT == MVT::f128) { 3485 assert(!Subtarget.hasP9Vector() && 3486 "SETCC for f128 is already legal under Power9!"); 3487 softenSetCCOperands(DAG, LHSVT, LHS, RHS, CC, dl, LHS, RHS, Chain, 3488 Op->getOpcode() == ISD::STRICT_FSETCCS); 3489 if (RHS.getNode()) 3490 LHS = DAG.getNode(ISD::SETCC, dl, Op.getValueType(), LHS, RHS, 3491 DAG.getCondCode(CC)); 3492 if (IsStrict) 3493 return DAG.getMergeValues({LHS, Chain}, dl); 3494 return LHS; 3495 } 3496 3497 assert(!IsStrict && "Don't know how to handle STRICT_FSETCC!"); 3498 3499 if (Op.getValueType() == MVT::v2i64) { 3500 // When the operands themselves are v2i64 values, we need to do something 3501 // special because VSX has no underlying comparison operations for these. 3502 if (LHS.getValueType() == MVT::v2i64) { 3503 // Equality can be handled by casting to the legal type for Altivec 3504 // comparisons, everything else needs to be expanded. 3505 if (CC != ISD::SETEQ && CC != ISD::SETNE) 3506 return SDValue(); 3507 SDValue SetCC32 = DAG.getSetCC( 3508 dl, MVT::v4i32, DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, LHS), 3509 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, RHS), CC); 3510 int ShuffV[] = {1, 0, 3, 2}; 3511 SDValue Shuff = 3512 DAG.getVectorShuffle(MVT::v4i32, dl, SetCC32, SetCC32, ShuffV); 3513 return DAG.getBitcast(MVT::v2i64, 3514 DAG.getNode(CC == ISD::SETEQ ? ISD::AND : ISD::OR, 3515 dl, MVT::v4i32, Shuff, SetCC32)); 3516 } 3517 3518 // We handle most of these in the usual way. 3519 return Op; 3520 } 3521 3522 // If we're comparing for equality to zero, expose the fact that this is 3523 // implemented as a ctlz/srl pair on ppc, so that the dag combiner can 3524 // fold the new nodes. 3525 if (SDValue V = lowerCmpEqZeroToCtlzSrl(Op, DAG)) 3526 return V; 3527 3528 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) { 3529 // Leave comparisons against 0 and -1 alone for now, since they're usually 3530 // optimized. FIXME: revisit this when we can custom lower all setcc 3531 // optimizations. 3532 if (C->isAllOnes() || C->isZero()) 3533 return SDValue(); 3534 } 3535 3536 // If we have an integer seteq/setne, turn it into a compare against zero 3537 // by xor'ing the rhs with the lhs, which is faster than setting a 3538 // condition register, reading it back out, and masking the correct bit. The 3539 // normal approach here uses sub to do this instead of xor. Using xor exposes 3540 // the result to other bit-twiddling opportunities. 3541 if (LHSVT.isInteger() && (CC == ISD::SETEQ || CC == ISD::SETNE)) { 3542 EVT VT = Op.getValueType(); 3543 SDValue Sub = DAG.getNode(ISD::XOR, dl, LHSVT, LHS, RHS); 3544 return DAG.getSetCC(dl, VT, Sub, DAG.getConstant(0, dl, LHSVT), CC); 3545 } 3546 return SDValue(); 3547 } 3548 3549 SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const { 3550 SDNode *Node = Op.getNode(); 3551 EVT VT = Node->getValueType(0); 3552 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 3553 SDValue InChain = Node->getOperand(0); 3554 SDValue VAListPtr = Node->getOperand(1); 3555 const Value *SV = cast<SrcValueSDNode>(Node->getOperand(2))->getValue(); 3556 SDLoc dl(Node); 3557 3558 assert(!Subtarget.isPPC64() && "LowerVAARG is PPC32 only"); 3559 3560 // gpr_index 3561 SDValue GprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain, 3562 VAListPtr, MachinePointerInfo(SV), MVT::i8); 3563 InChain = GprIndex.getValue(1); 3564 3565 if (VT == MVT::i64) { 3566 // Check if GprIndex is even 3567 SDValue GprAnd = DAG.getNode(ISD::AND, dl, MVT::i32, GprIndex, 3568 DAG.getConstant(1, dl, MVT::i32)); 3569 SDValue CC64 = DAG.getSetCC(dl, MVT::i32, GprAnd, 3570 DAG.getConstant(0, dl, MVT::i32), ISD::SETNE); 3571 SDValue GprIndexPlusOne = DAG.getNode(ISD::ADD, dl, MVT::i32, GprIndex, 3572 DAG.getConstant(1, dl, MVT::i32)); 3573 // Align GprIndex to be even if it isn't 3574 GprIndex = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC64, GprIndexPlusOne, 3575 GprIndex); 3576 } 3577 3578 // fpr index is 1 byte after gpr 3579 SDValue FprPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr, 3580 DAG.getConstant(1, dl, MVT::i32)); 3581 3582 // fpr 3583 SDValue FprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain, 3584 FprPtr, MachinePointerInfo(SV), MVT::i8); 3585 InChain = FprIndex.getValue(1); 3586 3587 SDValue RegSaveAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr, 3588 DAG.getConstant(8, dl, MVT::i32)); 3589 3590 SDValue OverflowAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr, 3591 DAG.getConstant(4, dl, MVT::i32)); 3592 3593 // areas 3594 SDValue OverflowArea = 3595 DAG.getLoad(MVT::i32, dl, InChain, OverflowAreaPtr, MachinePointerInfo()); 3596 InChain = OverflowArea.getValue(1); 3597 3598 SDValue RegSaveArea = 3599 DAG.getLoad(MVT::i32, dl, InChain, RegSaveAreaPtr, MachinePointerInfo()); 3600 InChain = RegSaveArea.getValue(1); 3601 3602 // select overflow_area if index > 8 3603 SDValue CC = DAG.getSetCC(dl, MVT::i32, VT.isInteger() ? GprIndex : FprIndex, 3604 DAG.getConstant(8, dl, MVT::i32), ISD::SETLT); 3605 3606 // adjustment constant gpr_index * 4/8 3607 SDValue RegConstant = DAG.getNode(ISD::MUL, dl, MVT::i32, 3608 VT.isInteger() ? GprIndex : FprIndex, 3609 DAG.getConstant(VT.isInteger() ? 4 : 8, dl, 3610 MVT::i32)); 3611 3612 // OurReg = RegSaveArea + RegConstant 3613 SDValue OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, RegSaveArea, 3614 RegConstant); 3615 3616 // Floating types are 32 bytes into RegSaveArea 3617 if (VT.isFloatingPoint()) 3618 OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, OurReg, 3619 DAG.getConstant(32, dl, MVT::i32)); 3620 3621 // increase {f,g}pr_index by 1 (or 2 if VT is i64) 3622 SDValue IndexPlus1 = DAG.getNode(ISD::ADD, dl, MVT::i32, 3623 VT.isInteger() ? GprIndex : FprIndex, 3624 DAG.getConstant(VT == MVT::i64 ? 2 : 1, dl, 3625 MVT::i32)); 3626 3627 InChain = DAG.getTruncStore(InChain, dl, IndexPlus1, 3628 VT.isInteger() ? VAListPtr : FprPtr, 3629 MachinePointerInfo(SV), MVT::i8); 3630 3631 // determine if we should load from reg_save_area or overflow_area 3632 SDValue Result = DAG.getNode(ISD::SELECT, dl, PtrVT, CC, OurReg, OverflowArea); 3633 3634 // increase overflow_area by 4/8 if gpr/fpr > 8 3635 SDValue OverflowAreaPlusN = DAG.getNode(ISD::ADD, dl, PtrVT, OverflowArea, 3636 DAG.getConstant(VT.isInteger() ? 4 : 8, 3637 dl, MVT::i32)); 3638 3639 OverflowArea = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC, OverflowArea, 3640 OverflowAreaPlusN); 3641 3642 InChain = DAG.getTruncStore(InChain, dl, OverflowArea, OverflowAreaPtr, 3643 MachinePointerInfo(), MVT::i32); 3644 3645 return DAG.getLoad(VT, dl, InChain, Result, MachinePointerInfo()); 3646 } 3647 3648 SDValue PPCTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const { 3649 assert(!Subtarget.isPPC64() && "LowerVACOPY is PPC32 only"); 3650 3651 // We have to copy the entire va_list struct: 3652 // 2*sizeof(char) + 2 Byte alignment + 2*sizeof(char*) = 12 Byte 3653 return DAG.getMemcpy(Op.getOperand(0), Op, Op.getOperand(1), Op.getOperand(2), 3654 DAG.getConstant(12, SDLoc(Op), MVT::i32), Align(8), 3655 false, true, false, MachinePointerInfo(), 3656 MachinePointerInfo()); 3657 } 3658 3659 SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op, 3660 SelectionDAG &DAG) const { 3661 if (Subtarget.isAIXABI()) 3662 report_fatal_error("ADJUST_TRAMPOLINE operation is not supported on AIX."); 3663 3664 return Op.getOperand(0); 3665 } 3666 3667 SDValue PPCTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const { 3668 MachineFunction &MF = DAG.getMachineFunction(); 3669 PPCFunctionInfo &MFI = *MF.getInfo<PPCFunctionInfo>(); 3670 3671 assert((Op.getOpcode() == ISD::INLINEASM || 3672 Op.getOpcode() == ISD::INLINEASM_BR) && 3673 "Expecting Inline ASM node."); 3674 3675 // If an LR store is already known to be required then there is not point in 3676 // checking this ASM as well. 3677 if (MFI.isLRStoreRequired()) 3678 return Op; 3679 3680 // Inline ASM nodes have an optional last operand that is an incoming Flag of 3681 // type MVT::Glue. We want to ignore this last operand if that is the case. 3682 unsigned NumOps = Op.getNumOperands(); 3683 if (Op.getOperand(NumOps - 1).getValueType() == MVT::Glue) 3684 --NumOps; 3685 3686 // Check all operands that may contain the LR. 3687 for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) { 3688 unsigned Flags = cast<ConstantSDNode>(Op.getOperand(i))->getZExtValue(); 3689 unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags); 3690 ++i; // Skip the ID value. 3691 3692 switch (InlineAsm::getKind(Flags)) { 3693 default: 3694 llvm_unreachable("Bad flags!"); 3695 case InlineAsm::Kind_RegUse: 3696 case InlineAsm::Kind_Imm: 3697 case InlineAsm::Kind_Mem: 3698 i += NumVals; 3699 break; 3700 case InlineAsm::Kind_Clobber: 3701 case InlineAsm::Kind_RegDef: 3702 case InlineAsm::Kind_RegDefEarlyClobber: { 3703 for (; NumVals; --NumVals, ++i) { 3704 Register Reg = cast<RegisterSDNode>(Op.getOperand(i))->getReg(); 3705 if (Reg != PPC::LR && Reg != PPC::LR8) 3706 continue; 3707 MFI.setLRStoreRequired(); 3708 return Op; 3709 } 3710 break; 3711 } 3712 } 3713 } 3714 3715 return Op; 3716 } 3717 3718 SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op, 3719 SelectionDAG &DAG) const { 3720 if (Subtarget.isAIXABI()) 3721 report_fatal_error("INIT_TRAMPOLINE operation is not supported on AIX."); 3722 3723 SDValue Chain = Op.getOperand(0); 3724 SDValue Trmp = Op.getOperand(1); // trampoline 3725 SDValue FPtr = Op.getOperand(2); // nested function 3726 SDValue Nest = Op.getOperand(3); // 'nest' parameter value 3727 SDLoc dl(Op); 3728 3729 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 3730 bool isPPC64 = (PtrVT == MVT::i64); 3731 Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext()); 3732 3733 TargetLowering::ArgListTy Args; 3734 TargetLowering::ArgListEntry Entry; 3735 3736 Entry.Ty = IntPtrTy; 3737 Entry.Node = Trmp; Args.push_back(Entry); 3738 3739 // TrampSize == (isPPC64 ? 48 : 40); 3740 Entry.Node = DAG.getConstant(isPPC64 ? 48 : 40, dl, 3741 isPPC64 ? MVT::i64 : MVT::i32); 3742 Args.push_back(Entry); 3743 3744 Entry.Node = FPtr; Args.push_back(Entry); 3745 Entry.Node = Nest; Args.push_back(Entry); 3746 3747 // Lower to a call to __trampoline_setup(Trmp, TrampSize, FPtr, ctx_reg) 3748 TargetLowering::CallLoweringInfo CLI(DAG); 3749 CLI.setDebugLoc(dl).setChain(Chain).setLibCallee( 3750 CallingConv::C, Type::getVoidTy(*DAG.getContext()), 3751 DAG.getExternalSymbol("__trampoline_setup", PtrVT), std::move(Args)); 3752 3753 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI); 3754 return CallResult.second; 3755 } 3756 3757 SDValue PPCTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const { 3758 MachineFunction &MF = DAG.getMachineFunction(); 3759 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 3760 EVT PtrVT = getPointerTy(MF.getDataLayout()); 3761 3762 SDLoc dl(Op); 3763 3764 if (Subtarget.isPPC64() || Subtarget.isAIXABI()) { 3765 // vastart just stores the address of the VarArgsFrameIndex slot into the 3766 // memory location argument. 3767 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 3768 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 3769 return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1), 3770 MachinePointerInfo(SV)); 3771 } 3772 3773 // For the 32-bit SVR4 ABI we follow the layout of the va_list struct. 3774 // We suppose the given va_list is already allocated. 3775 // 3776 // typedef struct { 3777 // char gpr; /* index into the array of 8 GPRs 3778 // * stored in the register save area 3779 // * gpr=0 corresponds to r3, 3780 // * gpr=1 to r4, etc. 3781 // */ 3782 // char fpr; /* index into the array of 8 FPRs 3783 // * stored in the register save area 3784 // * fpr=0 corresponds to f1, 3785 // * fpr=1 to f2, etc. 3786 // */ 3787 // char *overflow_arg_area; 3788 // /* location on stack that holds 3789 // * the next overflow argument 3790 // */ 3791 // char *reg_save_area; 3792 // /* where r3:r10 and f1:f8 (if saved) 3793 // * are stored 3794 // */ 3795 // } va_list[1]; 3796 3797 SDValue ArgGPR = DAG.getConstant(FuncInfo->getVarArgsNumGPR(), dl, MVT::i32); 3798 SDValue ArgFPR = DAG.getConstant(FuncInfo->getVarArgsNumFPR(), dl, MVT::i32); 3799 SDValue StackOffsetFI = DAG.getFrameIndex(FuncInfo->getVarArgsStackOffset(), 3800 PtrVT); 3801 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), 3802 PtrVT); 3803 3804 uint64_t FrameOffset = PtrVT.getSizeInBits()/8; 3805 SDValue ConstFrameOffset = DAG.getConstant(FrameOffset, dl, PtrVT); 3806 3807 uint64_t StackOffset = PtrVT.getSizeInBits()/8 - 1; 3808 SDValue ConstStackOffset = DAG.getConstant(StackOffset, dl, PtrVT); 3809 3810 uint64_t FPROffset = 1; 3811 SDValue ConstFPROffset = DAG.getConstant(FPROffset, dl, PtrVT); 3812 3813 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 3814 3815 // Store first byte : number of int regs 3816 SDValue firstStore = 3817 DAG.getTruncStore(Op.getOperand(0), dl, ArgGPR, Op.getOperand(1), 3818 MachinePointerInfo(SV), MVT::i8); 3819 uint64_t nextOffset = FPROffset; 3820 SDValue nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, Op.getOperand(1), 3821 ConstFPROffset); 3822 3823 // Store second byte : number of float regs 3824 SDValue secondStore = 3825 DAG.getTruncStore(firstStore, dl, ArgFPR, nextPtr, 3826 MachinePointerInfo(SV, nextOffset), MVT::i8); 3827 nextOffset += StackOffset; 3828 nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, nextPtr, ConstStackOffset); 3829 3830 // Store second word : arguments given on stack 3831 SDValue thirdStore = DAG.getStore(secondStore, dl, StackOffsetFI, nextPtr, 3832 MachinePointerInfo(SV, nextOffset)); 3833 nextOffset += FrameOffset; 3834 nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, nextPtr, ConstFrameOffset); 3835 3836 // Store third word : arguments given in registers 3837 return DAG.getStore(thirdStore, dl, FR, nextPtr, 3838 MachinePointerInfo(SV, nextOffset)); 3839 } 3840 3841 /// FPR - The set of FP registers that should be allocated for arguments 3842 /// on Darwin and AIX. 3843 static const MCPhysReg FPR[] = {PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, 3844 PPC::F6, PPC::F7, PPC::F8, PPC::F9, PPC::F10, 3845 PPC::F11, PPC::F12, PPC::F13}; 3846 3847 /// CalculateStackSlotSize - Calculates the size reserved for this argument on 3848 /// the stack. 3849 static unsigned CalculateStackSlotSize(EVT ArgVT, ISD::ArgFlagsTy Flags, 3850 unsigned PtrByteSize) { 3851 unsigned ArgSize = ArgVT.getStoreSize(); 3852 if (Flags.isByVal()) 3853 ArgSize = Flags.getByValSize(); 3854 3855 // Round up to multiples of the pointer size, except for array members, 3856 // which are always packed. 3857 if (!Flags.isInConsecutiveRegs()) 3858 ArgSize = ((ArgSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 3859 3860 return ArgSize; 3861 } 3862 3863 /// CalculateStackSlotAlignment - Calculates the alignment of this argument 3864 /// on the stack. 3865 static Align CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT, 3866 ISD::ArgFlagsTy Flags, 3867 unsigned PtrByteSize) { 3868 Align Alignment(PtrByteSize); 3869 3870 // Altivec parameters are padded to a 16 byte boundary. 3871 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 || 3872 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 || 3873 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 || 3874 ArgVT == MVT::v1i128 || ArgVT == MVT::f128) 3875 Alignment = Align(16); 3876 3877 // ByVal parameters are aligned as requested. 3878 if (Flags.isByVal()) { 3879 auto BVAlign = Flags.getNonZeroByValAlign(); 3880 if (BVAlign > PtrByteSize) { 3881 if (BVAlign.value() % PtrByteSize != 0) 3882 llvm_unreachable( 3883 "ByVal alignment is not a multiple of the pointer size"); 3884 3885 Alignment = BVAlign; 3886 } 3887 } 3888 3889 // Array members are always packed to their original alignment. 3890 if (Flags.isInConsecutiveRegs()) { 3891 // If the array member was split into multiple registers, the first 3892 // needs to be aligned to the size of the full type. (Except for 3893 // ppcf128, which is only aligned as its f64 components.) 3894 if (Flags.isSplit() && OrigVT != MVT::ppcf128) 3895 Alignment = Align(OrigVT.getStoreSize()); 3896 else 3897 Alignment = Align(ArgVT.getStoreSize()); 3898 } 3899 3900 return Alignment; 3901 } 3902 3903 /// CalculateStackSlotUsed - Return whether this argument will use its 3904 /// stack slot (instead of being passed in registers). ArgOffset, 3905 /// AvailableFPRs, and AvailableVRs must hold the current argument 3906 /// position, and will be updated to account for this argument. 3907 static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT, ISD::ArgFlagsTy Flags, 3908 unsigned PtrByteSize, unsigned LinkageSize, 3909 unsigned ParamAreaSize, unsigned &ArgOffset, 3910 unsigned &AvailableFPRs, 3911 unsigned &AvailableVRs) { 3912 bool UseMemory = false; 3913 3914 // Respect alignment of argument on the stack. 3915 Align Alignment = 3916 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize); 3917 ArgOffset = alignTo(ArgOffset, Alignment); 3918 // If there's no space left in the argument save area, we must 3919 // use memory (this check also catches zero-sized arguments). 3920 if (ArgOffset >= LinkageSize + ParamAreaSize) 3921 UseMemory = true; 3922 3923 // Allocate argument on the stack. 3924 ArgOffset += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize); 3925 if (Flags.isInConsecutiveRegsLast()) 3926 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 3927 // If we overran the argument save area, we must use memory 3928 // (this check catches arguments passed partially in memory) 3929 if (ArgOffset > LinkageSize + ParamAreaSize) 3930 UseMemory = true; 3931 3932 // However, if the argument is actually passed in an FPR or a VR, 3933 // we don't use memory after all. 3934 if (!Flags.isByVal()) { 3935 if (ArgVT == MVT::f32 || ArgVT == MVT::f64) 3936 if (AvailableFPRs > 0) { 3937 --AvailableFPRs; 3938 return false; 3939 } 3940 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 || 3941 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 || 3942 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 || 3943 ArgVT == MVT::v1i128 || ArgVT == MVT::f128) 3944 if (AvailableVRs > 0) { 3945 --AvailableVRs; 3946 return false; 3947 } 3948 } 3949 3950 return UseMemory; 3951 } 3952 3953 /// EnsureStackAlignment - Round stack frame size up from NumBytes to 3954 /// ensure minimum alignment required for target. 3955 static unsigned EnsureStackAlignment(const PPCFrameLowering *Lowering, 3956 unsigned NumBytes) { 3957 return alignTo(NumBytes, Lowering->getStackAlign()); 3958 } 3959 3960 SDValue PPCTargetLowering::LowerFormalArguments( 3961 SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 3962 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 3963 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 3964 if (Subtarget.isAIXABI()) 3965 return LowerFormalArguments_AIX(Chain, CallConv, isVarArg, Ins, dl, DAG, 3966 InVals); 3967 if (Subtarget.is64BitELFABI()) 3968 return LowerFormalArguments_64SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG, 3969 InVals); 3970 assert(Subtarget.is32BitELFABI()); 3971 return LowerFormalArguments_32SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG, 3972 InVals); 3973 } 3974 3975 SDValue PPCTargetLowering::LowerFormalArguments_32SVR4( 3976 SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 3977 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 3978 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 3979 3980 // 32-bit SVR4 ABI Stack Frame Layout: 3981 // +-----------------------------------+ 3982 // +--> | Back chain | 3983 // | +-----------------------------------+ 3984 // | | Floating-point register save area | 3985 // | +-----------------------------------+ 3986 // | | General register save area | 3987 // | +-----------------------------------+ 3988 // | | CR save word | 3989 // | +-----------------------------------+ 3990 // | | VRSAVE save word | 3991 // | +-----------------------------------+ 3992 // | | Alignment padding | 3993 // | +-----------------------------------+ 3994 // | | Vector register save area | 3995 // | +-----------------------------------+ 3996 // | | Local variable space | 3997 // | +-----------------------------------+ 3998 // | | Parameter list area | 3999 // | +-----------------------------------+ 4000 // | | LR save word | 4001 // | +-----------------------------------+ 4002 // SP--> +--- | Back chain | 4003 // +-----------------------------------+ 4004 // 4005 // Specifications: 4006 // System V Application Binary Interface PowerPC Processor Supplement 4007 // AltiVec Technology Programming Interface Manual 4008 4009 MachineFunction &MF = DAG.getMachineFunction(); 4010 MachineFrameInfo &MFI = MF.getFrameInfo(); 4011 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 4012 4013 EVT PtrVT = getPointerTy(MF.getDataLayout()); 4014 // Potential tail calls could cause overwriting of argument stack slots. 4015 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && 4016 (CallConv == CallingConv::Fast)); 4017 const Align PtrAlign(4); 4018 4019 // Assign locations to all of the incoming arguments. 4020 SmallVector<CCValAssign, 16> ArgLocs; 4021 PPCCCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs, 4022 *DAG.getContext()); 4023 4024 // Reserve space for the linkage area on the stack. 4025 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 4026 CCInfo.AllocateStack(LinkageSize, PtrAlign); 4027 if (useSoftFloat()) 4028 CCInfo.PreAnalyzeFormalArguments(Ins); 4029 4030 CCInfo.AnalyzeFormalArguments(Ins, CC_PPC32_SVR4); 4031 CCInfo.clearWasPPCF128(); 4032 4033 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 4034 CCValAssign &VA = ArgLocs[i]; 4035 4036 // Arguments stored in registers. 4037 if (VA.isRegLoc()) { 4038 const TargetRegisterClass *RC; 4039 EVT ValVT = VA.getValVT(); 4040 4041 switch (ValVT.getSimpleVT().SimpleTy) { 4042 default: 4043 llvm_unreachable("ValVT not supported by formal arguments Lowering"); 4044 case MVT::i1: 4045 case MVT::i32: 4046 RC = &PPC::GPRCRegClass; 4047 break; 4048 case MVT::f32: 4049 if (Subtarget.hasP8Vector()) 4050 RC = &PPC::VSSRCRegClass; 4051 else if (Subtarget.hasSPE()) 4052 RC = &PPC::GPRCRegClass; 4053 else 4054 RC = &PPC::F4RCRegClass; 4055 break; 4056 case MVT::f64: 4057 if (Subtarget.hasVSX()) 4058 RC = &PPC::VSFRCRegClass; 4059 else if (Subtarget.hasSPE()) 4060 // SPE passes doubles in GPR pairs. 4061 RC = &PPC::GPRCRegClass; 4062 else 4063 RC = &PPC::F8RCRegClass; 4064 break; 4065 case MVT::v16i8: 4066 case MVT::v8i16: 4067 case MVT::v4i32: 4068 RC = &PPC::VRRCRegClass; 4069 break; 4070 case MVT::v4f32: 4071 RC = &PPC::VRRCRegClass; 4072 break; 4073 case MVT::v2f64: 4074 case MVT::v2i64: 4075 RC = &PPC::VRRCRegClass; 4076 break; 4077 } 4078 4079 SDValue ArgValue; 4080 // Transform the arguments stored in physical registers into 4081 // virtual ones. 4082 if (VA.getLocVT() == MVT::f64 && Subtarget.hasSPE()) { 4083 assert(i + 1 < e && "No second half of double precision argument"); 4084 Register RegLo = MF.addLiveIn(VA.getLocReg(), RC); 4085 Register RegHi = MF.addLiveIn(ArgLocs[++i].getLocReg(), RC); 4086 SDValue ArgValueLo = DAG.getCopyFromReg(Chain, dl, RegLo, MVT::i32); 4087 SDValue ArgValueHi = DAG.getCopyFromReg(Chain, dl, RegHi, MVT::i32); 4088 if (!Subtarget.isLittleEndian()) 4089 std::swap (ArgValueLo, ArgValueHi); 4090 ArgValue = DAG.getNode(PPCISD::BUILD_SPE64, dl, MVT::f64, ArgValueLo, 4091 ArgValueHi); 4092 } else { 4093 Register Reg = MF.addLiveIn(VA.getLocReg(), RC); 4094 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, 4095 ValVT == MVT::i1 ? MVT::i32 : ValVT); 4096 if (ValVT == MVT::i1) 4097 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, ArgValue); 4098 } 4099 4100 InVals.push_back(ArgValue); 4101 } else { 4102 // Argument stored in memory. 4103 assert(VA.isMemLoc()); 4104 4105 // Get the extended size of the argument type in stack 4106 unsigned ArgSize = VA.getLocVT().getStoreSize(); 4107 // Get the actual size of the argument type 4108 unsigned ObjSize = VA.getValVT().getStoreSize(); 4109 unsigned ArgOffset = VA.getLocMemOffset(); 4110 // Stack objects in PPC32 are right justified. 4111 ArgOffset += ArgSize - ObjSize; 4112 int FI = MFI.CreateFixedObject(ArgSize, ArgOffset, isImmutable); 4113 4114 // Create load nodes to retrieve arguments from the stack. 4115 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 4116 InVals.push_back( 4117 DAG.getLoad(VA.getValVT(), dl, Chain, FIN, MachinePointerInfo())); 4118 } 4119 } 4120 4121 // Assign locations to all of the incoming aggregate by value arguments. 4122 // Aggregates passed by value are stored in the local variable space of the 4123 // caller's stack frame, right above the parameter list area. 4124 SmallVector<CCValAssign, 16> ByValArgLocs; 4125 CCState CCByValInfo(CallConv, isVarArg, DAG.getMachineFunction(), 4126 ByValArgLocs, *DAG.getContext()); 4127 4128 // Reserve stack space for the allocations in CCInfo. 4129 CCByValInfo.AllocateStack(CCInfo.getNextStackOffset(), PtrAlign); 4130 4131 CCByValInfo.AnalyzeFormalArguments(Ins, CC_PPC32_SVR4_ByVal); 4132 4133 // Area that is at least reserved in the caller of this function. 4134 unsigned MinReservedArea = CCByValInfo.getNextStackOffset(); 4135 MinReservedArea = std::max(MinReservedArea, LinkageSize); 4136 4137 // Set the size that is at least reserved in caller of this function. Tail 4138 // call optimized function's reserved stack space needs to be aligned so that 4139 // taking the difference between two stack areas will result in an aligned 4140 // stack. 4141 MinReservedArea = 4142 EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea); 4143 FuncInfo->setMinReservedArea(MinReservedArea); 4144 4145 SmallVector<SDValue, 8> MemOps; 4146 4147 // If the function takes variable number of arguments, make a frame index for 4148 // the start of the first vararg value... for expansion of llvm.va_start. 4149 if (isVarArg) { 4150 static const MCPhysReg GPArgRegs[] = { 4151 PPC::R3, PPC::R4, PPC::R5, PPC::R6, 4152 PPC::R7, PPC::R8, PPC::R9, PPC::R10, 4153 }; 4154 const unsigned NumGPArgRegs = array_lengthof(GPArgRegs); 4155 4156 static const MCPhysReg FPArgRegs[] = { 4157 PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7, 4158 PPC::F8 4159 }; 4160 unsigned NumFPArgRegs = array_lengthof(FPArgRegs); 4161 4162 if (useSoftFloat() || hasSPE()) 4163 NumFPArgRegs = 0; 4164 4165 FuncInfo->setVarArgsNumGPR(CCInfo.getFirstUnallocated(GPArgRegs)); 4166 FuncInfo->setVarArgsNumFPR(CCInfo.getFirstUnallocated(FPArgRegs)); 4167 4168 // Make room for NumGPArgRegs and NumFPArgRegs. 4169 int Depth = NumGPArgRegs * PtrVT.getSizeInBits()/8 + 4170 NumFPArgRegs * MVT(MVT::f64).getSizeInBits()/8; 4171 4172 FuncInfo->setVarArgsStackOffset( 4173 MFI.CreateFixedObject(PtrVT.getSizeInBits()/8, 4174 CCInfo.getNextStackOffset(), true)); 4175 4176 FuncInfo->setVarArgsFrameIndex( 4177 MFI.CreateStackObject(Depth, Align(8), false)); 4178 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 4179 4180 // The fixed integer arguments of a variadic function are stored to the 4181 // VarArgsFrameIndex on the stack so that they may be loaded by 4182 // dereferencing the result of va_next. 4183 for (unsigned GPRIndex = 0; GPRIndex != NumGPArgRegs; ++GPRIndex) { 4184 // Get an existing live-in vreg, or add a new one. 4185 Register VReg = MF.getRegInfo().getLiveInVirtReg(GPArgRegs[GPRIndex]); 4186 if (!VReg) 4187 VReg = MF.addLiveIn(GPArgRegs[GPRIndex], &PPC::GPRCRegClass); 4188 4189 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 4190 SDValue Store = 4191 DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo()); 4192 MemOps.push_back(Store); 4193 // Increment the address by four for the next argument to store 4194 SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, dl, PtrVT); 4195 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); 4196 } 4197 4198 // FIXME 32-bit SVR4: We only need to save FP argument registers if CR bit 6 4199 // is set. 4200 // The double arguments are stored to the VarArgsFrameIndex 4201 // on the stack. 4202 for (unsigned FPRIndex = 0; FPRIndex != NumFPArgRegs; ++FPRIndex) { 4203 // Get an existing live-in vreg, or add a new one. 4204 Register VReg = MF.getRegInfo().getLiveInVirtReg(FPArgRegs[FPRIndex]); 4205 if (!VReg) 4206 VReg = MF.addLiveIn(FPArgRegs[FPRIndex], &PPC::F8RCRegClass); 4207 4208 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::f64); 4209 SDValue Store = 4210 DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo()); 4211 MemOps.push_back(Store); 4212 // Increment the address by eight for the next argument to store 4213 SDValue PtrOff = DAG.getConstant(MVT(MVT::f64).getSizeInBits()/8, dl, 4214 PtrVT); 4215 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); 4216 } 4217 } 4218 4219 if (!MemOps.empty()) 4220 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); 4221 4222 return Chain; 4223 } 4224 4225 // PPC64 passes i8, i16, and i32 values in i64 registers. Promote 4226 // value to MVT::i64 and then truncate to the correct register size. 4227 SDValue PPCTargetLowering::extendArgForPPC64(ISD::ArgFlagsTy Flags, 4228 EVT ObjectVT, SelectionDAG &DAG, 4229 SDValue ArgVal, 4230 const SDLoc &dl) const { 4231 if (Flags.isSExt()) 4232 ArgVal = DAG.getNode(ISD::AssertSext, dl, MVT::i64, ArgVal, 4233 DAG.getValueType(ObjectVT)); 4234 else if (Flags.isZExt()) 4235 ArgVal = DAG.getNode(ISD::AssertZext, dl, MVT::i64, ArgVal, 4236 DAG.getValueType(ObjectVT)); 4237 4238 return DAG.getNode(ISD::TRUNCATE, dl, ObjectVT, ArgVal); 4239 } 4240 4241 SDValue PPCTargetLowering::LowerFormalArguments_64SVR4( 4242 SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 4243 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 4244 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 4245 // TODO: add description of PPC stack frame format, or at least some docs. 4246 // 4247 bool isELFv2ABI = Subtarget.isELFv2ABI(); 4248 bool isLittleEndian = Subtarget.isLittleEndian(); 4249 MachineFunction &MF = DAG.getMachineFunction(); 4250 MachineFrameInfo &MFI = MF.getFrameInfo(); 4251 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 4252 4253 assert(!(CallConv == CallingConv::Fast && isVarArg) && 4254 "fastcc not supported on varargs functions"); 4255 4256 EVT PtrVT = getPointerTy(MF.getDataLayout()); 4257 // Potential tail calls could cause overwriting of argument stack slots. 4258 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && 4259 (CallConv == CallingConv::Fast)); 4260 unsigned PtrByteSize = 8; 4261 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 4262 4263 static const MCPhysReg GPR[] = { 4264 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 4265 PPC::X7, PPC::X8, PPC::X9, PPC::X10, 4266 }; 4267 static const MCPhysReg VR[] = { 4268 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, 4269 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 4270 }; 4271 4272 const unsigned Num_GPR_Regs = array_lengthof(GPR); 4273 const unsigned Num_FPR_Regs = useSoftFloat() ? 0 : 13; 4274 const unsigned Num_VR_Regs = array_lengthof(VR); 4275 4276 // Do a first pass over the arguments to determine whether the ABI 4277 // guarantees that our caller has allocated the parameter save area 4278 // on its stack frame. In the ELFv1 ABI, this is always the case; 4279 // in the ELFv2 ABI, it is true if this is a vararg function or if 4280 // any parameter is located in a stack slot. 4281 4282 bool HasParameterArea = !isELFv2ABI || isVarArg; 4283 unsigned ParamAreaSize = Num_GPR_Regs * PtrByteSize; 4284 unsigned NumBytes = LinkageSize; 4285 unsigned AvailableFPRs = Num_FPR_Regs; 4286 unsigned AvailableVRs = Num_VR_Regs; 4287 for (unsigned i = 0, e = Ins.size(); i != e; ++i) { 4288 if (Ins[i].Flags.isNest()) 4289 continue; 4290 4291 if (CalculateStackSlotUsed(Ins[i].VT, Ins[i].ArgVT, Ins[i].Flags, 4292 PtrByteSize, LinkageSize, ParamAreaSize, 4293 NumBytes, AvailableFPRs, AvailableVRs)) 4294 HasParameterArea = true; 4295 } 4296 4297 // Add DAG nodes to load the arguments or copy them out of registers. On 4298 // entry to a function on PPC, the arguments start after the linkage area, 4299 // although the first ones are often in registers. 4300 4301 unsigned ArgOffset = LinkageSize; 4302 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; 4303 SmallVector<SDValue, 8> MemOps; 4304 Function::const_arg_iterator FuncArg = MF.getFunction().arg_begin(); 4305 unsigned CurArgIdx = 0; 4306 for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) { 4307 SDValue ArgVal; 4308 bool needsLoad = false; 4309 EVT ObjectVT = Ins[ArgNo].VT; 4310 EVT OrigVT = Ins[ArgNo].ArgVT; 4311 unsigned ObjSize = ObjectVT.getStoreSize(); 4312 unsigned ArgSize = ObjSize; 4313 ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags; 4314 if (Ins[ArgNo].isOrigArg()) { 4315 std::advance(FuncArg, Ins[ArgNo].getOrigArgIndex() - CurArgIdx); 4316 CurArgIdx = Ins[ArgNo].getOrigArgIndex(); 4317 } 4318 // We re-align the argument offset for each argument, except when using the 4319 // fast calling convention, when we need to make sure we do that only when 4320 // we'll actually use a stack slot. 4321 unsigned CurArgOffset; 4322 Align Alignment; 4323 auto ComputeArgOffset = [&]() { 4324 /* Respect alignment of argument on the stack. */ 4325 Alignment = 4326 CalculateStackSlotAlignment(ObjectVT, OrigVT, Flags, PtrByteSize); 4327 ArgOffset = alignTo(ArgOffset, Alignment); 4328 CurArgOffset = ArgOffset; 4329 }; 4330 4331 if (CallConv != CallingConv::Fast) { 4332 ComputeArgOffset(); 4333 4334 /* Compute GPR index associated with argument offset. */ 4335 GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize; 4336 GPR_idx = std::min(GPR_idx, Num_GPR_Regs); 4337 } 4338 4339 // FIXME the codegen can be much improved in some cases. 4340 // We do not have to keep everything in memory. 4341 if (Flags.isByVal()) { 4342 assert(Ins[ArgNo].isOrigArg() && "Byval arguments cannot be implicit"); 4343 4344 if (CallConv == CallingConv::Fast) 4345 ComputeArgOffset(); 4346 4347 // ObjSize is the true size, ArgSize rounded up to multiple of registers. 4348 ObjSize = Flags.getByValSize(); 4349 ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 4350 // Empty aggregate parameters do not take up registers. Examples: 4351 // struct { } a; 4352 // union { } b; 4353 // int c[0]; 4354 // etc. However, we have to provide a place-holder in InVals, so 4355 // pretend we have an 8-byte item at the current address for that 4356 // purpose. 4357 if (!ObjSize) { 4358 int FI = MFI.CreateFixedObject(PtrByteSize, ArgOffset, true); 4359 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 4360 InVals.push_back(FIN); 4361 continue; 4362 } 4363 4364 // Create a stack object covering all stack doublewords occupied 4365 // by the argument. If the argument is (fully or partially) on 4366 // the stack, or if the argument is fully in registers but the 4367 // caller has allocated the parameter save anyway, we can refer 4368 // directly to the caller's stack frame. Otherwise, create a 4369 // local copy in our own frame. 4370 int FI; 4371 if (HasParameterArea || 4372 ArgSize + ArgOffset > LinkageSize + Num_GPR_Regs * PtrByteSize) 4373 FI = MFI.CreateFixedObject(ArgSize, ArgOffset, false, true); 4374 else 4375 FI = MFI.CreateStackObject(ArgSize, Alignment, false); 4376 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 4377 4378 // Handle aggregates smaller than 8 bytes. 4379 if (ObjSize < PtrByteSize) { 4380 // The value of the object is its address, which differs from the 4381 // address of the enclosing doubleword on big-endian systems. 4382 SDValue Arg = FIN; 4383 if (!isLittleEndian) { 4384 SDValue ArgOff = DAG.getConstant(PtrByteSize - ObjSize, dl, PtrVT); 4385 Arg = DAG.getNode(ISD::ADD, dl, ArgOff.getValueType(), Arg, ArgOff); 4386 } 4387 InVals.push_back(Arg); 4388 4389 if (GPR_idx != Num_GPR_Regs) { 4390 Register VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass); 4391 FuncInfo->addLiveInAttr(VReg, Flags); 4392 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 4393 EVT ObjType = EVT::getIntegerVT(*DAG.getContext(), ObjSize * 8); 4394 SDValue Store = 4395 DAG.getTruncStore(Val.getValue(1), dl, Val, Arg, 4396 MachinePointerInfo(&*FuncArg), ObjType); 4397 MemOps.push_back(Store); 4398 } 4399 // Whether we copied from a register or not, advance the offset 4400 // into the parameter save area by a full doubleword. 4401 ArgOffset += PtrByteSize; 4402 continue; 4403 } 4404 4405 // The value of the object is its address, which is the address of 4406 // its first stack doubleword. 4407 InVals.push_back(FIN); 4408 4409 // Store whatever pieces of the object are in registers to memory. 4410 for (unsigned j = 0; j < ArgSize; j += PtrByteSize) { 4411 if (GPR_idx == Num_GPR_Regs) 4412 break; 4413 4414 Register VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 4415 FuncInfo->addLiveInAttr(VReg, Flags); 4416 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 4417 SDValue Addr = FIN; 4418 if (j) { 4419 SDValue Off = DAG.getConstant(j, dl, PtrVT); 4420 Addr = DAG.getNode(ISD::ADD, dl, Off.getValueType(), Addr, Off); 4421 } 4422 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, Addr, 4423 MachinePointerInfo(&*FuncArg, j)); 4424 MemOps.push_back(Store); 4425 ++GPR_idx; 4426 } 4427 ArgOffset += ArgSize; 4428 continue; 4429 } 4430 4431 switch (ObjectVT.getSimpleVT().SimpleTy) { 4432 default: llvm_unreachable("Unhandled argument type!"); 4433 case MVT::i1: 4434 case MVT::i32: 4435 case MVT::i64: 4436 if (Flags.isNest()) { 4437 // The 'nest' parameter, if any, is passed in R11. 4438 Register VReg = MF.addLiveIn(PPC::X11, &PPC::G8RCRegClass); 4439 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); 4440 4441 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1) 4442 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl); 4443 4444 break; 4445 } 4446 4447 // These can be scalar arguments or elements of an integer array type 4448 // passed directly. Clang may use those instead of "byval" aggregate 4449 // types to avoid forcing arguments to memory unnecessarily. 4450 if (GPR_idx != Num_GPR_Regs) { 4451 Register VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass); 4452 FuncInfo->addLiveInAttr(VReg, Flags); 4453 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); 4454 4455 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1) 4456 // PPC64 passes i8, i16, and i32 values in i64 registers. Promote 4457 // value to MVT::i64 and then truncate to the correct register size. 4458 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl); 4459 } else { 4460 if (CallConv == CallingConv::Fast) 4461 ComputeArgOffset(); 4462 4463 needsLoad = true; 4464 ArgSize = PtrByteSize; 4465 } 4466 if (CallConv != CallingConv::Fast || needsLoad) 4467 ArgOffset += 8; 4468 break; 4469 4470 case MVT::f32: 4471 case MVT::f64: 4472 // These can be scalar arguments or elements of a float array type 4473 // passed directly. The latter are used to implement ELFv2 homogenous 4474 // float aggregates. 4475 if (FPR_idx != Num_FPR_Regs) { 4476 unsigned VReg; 4477 4478 if (ObjectVT == MVT::f32) 4479 VReg = MF.addLiveIn(FPR[FPR_idx], 4480 Subtarget.hasP8Vector() 4481 ? &PPC::VSSRCRegClass 4482 : &PPC::F4RCRegClass); 4483 else 4484 VReg = MF.addLiveIn(FPR[FPR_idx], Subtarget.hasVSX() 4485 ? &PPC::VSFRCRegClass 4486 : &PPC::F8RCRegClass); 4487 4488 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); 4489 ++FPR_idx; 4490 } else if (GPR_idx != Num_GPR_Regs && CallConv != CallingConv::Fast) { 4491 // FIXME: We may want to re-enable this for CallingConv::Fast on the P8 4492 // once we support fp <-> gpr moves. 4493 4494 // This can only ever happen in the presence of f32 array types, 4495 // since otherwise we never run out of FPRs before running out 4496 // of GPRs. 4497 Register VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass); 4498 FuncInfo->addLiveInAttr(VReg, Flags); 4499 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); 4500 4501 if (ObjectVT == MVT::f32) { 4502 if ((ArgOffset % PtrByteSize) == (isLittleEndian ? 4 : 0)) 4503 ArgVal = DAG.getNode(ISD::SRL, dl, MVT::i64, ArgVal, 4504 DAG.getConstant(32, dl, MVT::i32)); 4505 ArgVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, ArgVal); 4506 } 4507 4508 ArgVal = DAG.getNode(ISD::BITCAST, dl, ObjectVT, ArgVal); 4509 } else { 4510 if (CallConv == CallingConv::Fast) 4511 ComputeArgOffset(); 4512 4513 needsLoad = true; 4514 } 4515 4516 // When passing an array of floats, the array occupies consecutive 4517 // space in the argument area; only round up to the next doubleword 4518 // at the end of the array. Otherwise, each float takes 8 bytes. 4519 if (CallConv != CallingConv::Fast || needsLoad) { 4520 ArgSize = Flags.isInConsecutiveRegs() ? ObjSize : PtrByteSize; 4521 ArgOffset += ArgSize; 4522 if (Flags.isInConsecutiveRegsLast()) 4523 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 4524 } 4525 break; 4526 case MVT::v4f32: 4527 case MVT::v4i32: 4528 case MVT::v8i16: 4529 case MVT::v16i8: 4530 case MVT::v2f64: 4531 case MVT::v2i64: 4532 case MVT::v1i128: 4533 case MVT::f128: 4534 // These can be scalar arguments or elements of a vector array type 4535 // passed directly. The latter are used to implement ELFv2 homogenous 4536 // vector aggregates. 4537 if (VR_idx != Num_VR_Regs) { 4538 Register VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass); 4539 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); 4540 ++VR_idx; 4541 } else { 4542 if (CallConv == CallingConv::Fast) 4543 ComputeArgOffset(); 4544 needsLoad = true; 4545 } 4546 if (CallConv != CallingConv::Fast || needsLoad) 4547 ArgOffset += 16; 4548 break; 4549 } 4550 4551 // We need to load the argument to a virtual register if we determined 4552 // above that we ran out of physical registers of the appropriate type. 4553 if (needsLoad) { 4554 if (ObjSize < ArgSize && !isLittleEndian) 4555 CurArgOffset += ArgSize - ObjSize; 4556 int FI = MFI.CreateFixedObject(ObjSize, CurArgOffset, isImmutable); 4557 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 4558 ArgVal = DAG.getLoad(ObjectVT, dl, Chain, FIN, MachinePointerInfo()); 4559 } 4560 4561 InVals.push_back(ArgVal); 4562 } 4563 4564 // Area that is at least reserved in the caller of this function. 4565 unsigned MinReservedArea; 4566 if (HasParameterArea) 4567 MinReservedArea = std::max(ArgOffset, LinkageSize + 8 * PtrByteSize); 4568 else 4569 MinReservedArea = LinkageSize; 4570 4571 // Set the size that is at least reserved in caller of this function. Tail 4572 // call optimized functions' reserved stack space needs to be aligned so that 4573 // taking the difference between two stack areas will result in an aligned 4574 // stack. 4575 MinReservedArea = 4576 EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea); 4577 FuncInfo->setMinReservedArea(MinReservedArea); 4578 4579 // If the function takes variable number of arguments, make a frame index for 4580 // the start of the first vararg value... for expansion of llvm.va_start. 4581 // On ELFv2ABI spec, it writes: 4582 // C programs that are intended to be *portable* across different compilers 4583 // and architectures must use the header file <stdarg.h> to deal with variable 4584 // argument lists. 4585 if (isVarArg && MFI.hasVAStart()) { 4586 int Depth = ArgOffset; 4587 4588 FuncInfo->setVarArgsFrameIndex( 4589 MFI.CreateFixedObject(PtrByteSize, Depth, true)); 4590 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 4591 4592 // If this function is vararg, store any remaining integer argument regs 4593 // to their spots on the stack so that they may be loaded by dereferencing 4594 // the result of va_next. 4595 for (GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize; 4596 GPR_idx < Num_GPR_Regs; ++GPR_idx) { 4597 Register VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 4598 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 4599 SDValue Store = 4600 DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo()); 4601 MemOps.push_back(Store); 4602 // Increment the address by four for the next argument to store 4603 SDValue PtrOff = DAG.getConstant(PtrByteSize, dl, PtrVT); 4604 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); 4605 } 4606 } 4607 4608 if (!MemOps.empty()) 4609 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); 4610 4611 return Chain; 4612 } 4613 4614 /// CalculateTailCallSPDiff - Get the amount the stack pointer has to be 4615 /// adjusted to accommodate the arguments for the tailcall. 4616 static int CalculateTailCallSPDiff(SelectionDAG& DAG, bool isTailCall, 4617 unsigned ParamSize) { 4618 4619 if (!isTailCall) return 0; 4620 4621 PPCFunctionInfo *FI = DAG.getMachineFunction().getInfo<PPCFunctionInfo>(); 4622 unsigned CallerMinReservedArea = FI->getMinReservedArea(); 4623 int SPDiff = (int)CallerMinReservedArea - (int)ParamSize; 4624 // Remember only if the new adjustment is bigger. 4625 if (SPDiff < FI->getTailCallSPDelta()) 4626 FI->setTailCallSPDelta(SPDiff); 4627 4628 return SPDiff; 4629 } 4630 4631 static bool isFunctionGlobalAddress(SDValue Callee); 4632 4633 static bool callsShareTOCBase(const Function *Caller, SDValue Callee, 4634 const TargetMachine &TM) { 4635 // It does not make sense to call callsShareTOCBase() with a caller that 4636 // is PC Relative since PC Relative callers do not have a TOC. 4637 #ifndef NDEBUG 4638 const PPCSubtarget *STICaller = &TM.getSubtarget<PPCSubtarget>(*Caller); 4639 assert(!STICaller->isUsingPCRelativeCalls() && 4640 "PC Relative callers do not have a TOC and cannot share a TOC Base"); 4641 #endif 4642 4643 // Callee is either a GlobalAddress or an ExternalSymbol. ExternalSymbols 4644 // don't have enough information to determine if the caller and callee share 4645 // the same TOC base, so we have to pessimistically assume they don't for 4646 // correctness. 4647 GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee); 4648 if (!G) 4649 return false; 4650 4651 const GlobalValue *GV = G->getGlobal(); 4652 4653 // If the callee is preemptable, then the static linker will use a plt-stub 4654 // which saves the toc to the stack, and needs a nop after the call 4655 // instruction to convert to a toc-restore. 4656 if (!TM.shouldAssumeDSOLocal(*Caller->getParent(), GV)) 4657 return false; 4658 4659 // Functions with PC Relative enabled may clobber the TOC in the same DSO. 4660 // We may need a TOC restore in the situation where the caller requires a 4661 // valid TOC but the callee is PC Relative and does not. 4662 const Function *F = dyn_cast<Function>(GV); 4663 const GlobalAlias *Alias = dyn_cast<GlobalAlias>(GV); 4664 4665 // If we have an Alias we can try to get the function from there. 4666 if (Alias) { 4667 const GlobalObject *GlobalObj = Alias->getAliaseeObject(); 4668 F = dyn_cast<Function>(GlobalObj); 4669 } 4670 4671 // If we still have no valid function pointer we do not have enough 4672 // information to determine if the callee uses PC Relative calls so we must 4673 // assume that it does. 4674 if (!F) 4675 return false; 4676 4677 // If the callee uses PC Relative we cannot guarantee that the callee won't 4678 // clobber the TOC of the caller and so we must assume that the two 4679 // functions do not share a TOC base. 4680 const PPCSubtarget *STICallee = &TM.getSubtarget<PPCSubtarget>(*F); 4681 if (STICallee->isUsingPCRelativeCalls()) 4682 return false; 4683 4684 // If the GV is not a strong definition then we need to assume it can be 4685 // replaced by another function at link time. The function that replaces 4686 // it may not share the same TOC as the caller since the callee may be 4687 // replaced by a PC Relative version of the same function. 4688 if (!GV->isStrongDefinitionForLinker()) 4689 return false; 4690 4691 // The medium and large code models are expected to provide a sufficiently 4692 // large TOC to provide all data addressing needs of a module with a 4693 // single TOC. 4694 if (CodeModel::Medium == TM.getCodeModel() || 4695 CodeModel::Large == TM.getCodeModel()) 4696 return true; 4697 4698 // Any explicitly-specified sections and section prefixes must also match. 4699 // Also, if we're using -ffunction-sections, then each function is always in 4700 // a different section (the same is true for COMDAT functions). 4701 if (TM.getFunctionSections() || GV->hasComdat() || Caller->hasComdat() || 4702 GV->getSection() != Caller->getSection()) 4703 return false; 4704 if (const auto *F = dyn_cast<Function>(GV)) { 4705 if (F->getSectionPrefix() != Caller->getSectionPrefix()) 4706 return false; 4707 } 4708 4709 return true; 4710 } 4711 4712 static bool 4713 needStackSlotPassParameters(const PPCSubtarget &Subtarget, 4714 const SmallVectorImpl<ISD::OutputArg> &Outs) { 4715 assert(Subtarget.is64BitELFABI()); 4716 4717 const unsigned PtrByteSize = 8; 4718 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 4719 4720 static const MCPhysReg GPR[] = { 4721 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 4722 PPC::X7, PPC::X8, PPC::X9, PPC::X10, 4723 }; 4724 static const MCPhysReg VR[] = { 4725 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, 4726 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 4727 }; 4728 4729 const unsigned NumGPRs = array_lengthof(GPR); 4730 const unsigned NumFPRs = 13; 4731 const unsigned NumVRs = array_lengthof(VR); 4732 const unsigned ParamAreaSize = NumGPRs * PtrByteSize; 4733 4734 unsigned NumBytes = LinkageSize; 4735 unsigned AvailableFPRs = NumFPRs; 4736 unsigned AvailableVRs = NumVRs; 4737 4738 for (const ISD::OutputArg& Param : Outs) { 4739 if (Param.Flags.isNest()) continue; 4740 4741 if (CalculateStackSlotUsed(Param.VT, Param.ArgVT, Param.Flags, PtrByteSize, 4742 LinkageSize, ParamAreaSize, NumBytes, 4743 AvailableFPRs, AvailableVRs)) 4744 return true; 4745 } 4746 return false; 4747 } 4748 4749 static bool hasSameArgumentList(const Function *CallerFn, const CallBase &CB) { 4750 if (CB.arg_size() != CallerFn->arg_size()) 4751 return false; 4752 4753 auto CalleeArgIter = CB.arg_begin(); 4754 auto CalleeArgEnd = CB.arg_end(); 4755 Function::const_arg_iterator CallerArgIter = CallerFn->arg_begin(); 4756 4757 for (; CalleeArgIter != CalleeArgEnd; ++CalleeArgIter, ++CallerArgIter) { 4758 const Value* CalleeArg = *CalleeArgIter; 4759 const Value* CallerArg = &(*CallerArgIter); 4760 if (CalleeArg == CallerArg) 4761 continue; 4762 4763 // e.g. @caller([4 x i64] %a, [4 x i64] %b) { 4764 // tail call @callee([4 x i64] undef, [4 x i64] %b) 4765 // } 4766 // 1st argument of callee is undef and has the same type as caller. 4767 if (CalleeArg->getType() == CallerArg->getType() && 4768 isa<UndefValue>(CalleeArg)) 4769 continue; 4770 4771 return false; 4772 } 4773 4774 return true; 4775 } 4776 4777 // Returns true if TCO is possible between the callers and callees 4778 // calling conventions. 4779 static bool 4780 areCallingConvEligibleForTCO_64SVR4(CallingConv::ID CallerCC, 4781 CallingConv::ID CalleeCC) { 4782 // Tail calls are possible with fastcc and ccc. 4783 auto isTailCallableCC = [] (CallingConv::ID CC){ 4784 return CC == CallingConv::C || CC == CallingConv::Fast; 4785 }; 4786 if (!isTailCallableCC(CallerCC) || !isTailCallableCC(CalleeCC)) 4787 return false; 4788 4789 // We can safely tail call both fastcc and ccc callees from a c calling 4790 // convention caller. If the caller is fastcc, we may have less stack space 4791 // than a non-fastcc caller with the same signature so disable tail-calls in 4792 // that case. 4793 return CallerCC == CallingConv::C || CallerCC == CalleeCC; 4794 } 4795 4796 bool PPCTargetLowering::IsEligibleForTailCallOptimization_64SVR4( 4797 SDValue Callee, CallingConv::ID CalleeCC, const CallBase *CB, bool isVarArg, 4798 const SmallVectorImpl<ISD::OutputArg> &Outs, 4799 const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const { 4800 bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt; 4801 4802 if (DisableSCO && !TailCallOpt) return false; 4803 4804 // Variadic argument functions are not supported. 4805 if (isVarArg) return false; 4806 4807 auto &Caller = DAG.getMachineFunction().getFunction(); 4808 // Check that the calling conventions are compatible for tco. 4809 if (!areCallingConvEligibleForTCO_64SVR4(Caller.getCallingConv(), CalleeCC)) 4810 return false; 4811 4812 // Caller contains any byval parameter is not supported. 4813 if (any_of(Ins, [](const ISD::InputArg &IA) { return IA.Flags.isByVal(); })) 4814 return false; 4815 4816 // Callee contains any byval parameter is not supported, too. 4817 // Note: This is a quick work around, because in some cases, e.g. 4818 // caller's stack size > callee's stack size, we are still able to apply 4819 // sibling call optimization. For example, gcc is able to do SCO for caller1 4820 // in the following example, but not for caller2. 4821 // struct test { 4822 // long int a; 4823 // char ary[56]; 4824 // } gTest; 4825 // __attribute__((noinline)) int callee(struct test v, struct test *b) { 4826 // b->a = v.a; 4827 // return 0; 4828 // } 4829 // void caller1(struct test a, struct test c, struct test *b) { 4830 // callee(gTest, b); } 4831 // void caller2(struct test *b) { callee(gTest, b); } 4832 if (any_of(Outs, [](const ISD::OutputArg& OA) { return OA.Flags.isByVal(); })) 4833 return false; 4834 4835 // If callee and caller use different calling conventions, we cannot pass 4836 // parameters on stack since offsets for the parameter area may be different. 4837 if (Caller.getCallingConv() != CalleeCC && 4838 needStackSlotPassParameters(Subtarget, Outs)) 4839 return false; 4840 4841 // All variants of 64-bit ELF ABIs without PC-Relative addressing require that 4842 // the caller and callee share the same TOC for TCO/SCO. If the caller and 4843 // callee potentially have different TOC bases then we cannot tail call since 4844 // we need to restore the TOC pointer after the call. 4845 // ref: https://bugzilla.mozilla.org/show_bug.cgi?id=973977 4846 // We cannot guarantee this for indirect calls or calls to external functions. 4847 // When PC-Relative addressing is used, the concept of the TOC is no longer 4848 // applicable so this check is not required. 4849 // Check first for indirect calls. 4850 if (!Subtarget.isUsingPCRelativeCalls() && 4851 !isFunctionGlobalAddress(Callee) && !isa<ExternalSymbolSDNode>(Callee)) 4852 return false; 4853 4854 // Check if we share the TOC base. 4855 if (!Subtarget.isUsingPCRelativeCalls() && 4856 !callsShareTOCBase(&Caller, Callee, getTargetMachine())) 4857 return false; 4858 4859 // TCO allows altering callee ABI, so we don't have to check further. 4860 if (CalleeCC == CallingConv::Fast && TailCallOpt) 4861 return true; 4862 4863 if (DisableSCO) return false; 4864 4865 // If callee use the same argument list that caller is using, then we can 4866 // apply SCO on this case. If it is not, then we need to check if callee needs 4867 // stack for passing arguments. 4868 // PC Relative tail calls may not have a CallBase. 4869 // If there is no CallBase we cannot verify if we have the same argument 4870 // list so assume that we don't have the same argument list. 4871 if (CB && !hasSameArgumentList(&Caller, *CB) && 4872 needStackSlotPassParameters(Subtarget, Outs)) 4873 return false; 4874 else if (!CB && needStackSlotPassParameters(Subtarget, Outs)) 4875 return false; 4876 4877 return true; 4878 } 4879 4880 /// IsEligibleForTailCallOptimization - Check whether the call is eligible 4881 /// for tail call optimization. Targets which want to do tail call 4882 /// optimization should implement this function. 4883 bool 4884 PPCTargetLowering::IsEligibleForTailCallOptimization(SDValue Callee, 4885 CallingConv::ID CalleeCC, 4886 bool isVarArg, 4887 const SmallVectorImpl<ISD::InputArg> &Ins, 4888 SelectionDAG& DAG) const { 4889 if (!getTargetMachine().Options.GuaranteedTailCallOpt) 4890 return false; 4891 4892 // Variable argument functions are not supported. 4893 if (isVarArg) 4894 return false; 4895 4896 MachineFunction &MF = DAG.getMachineFunction(); 4897 CallingConv::ID CallerCC = MF.getFunction().getCallingConv(); 4898 if (CalleeCC == CallingConv::Fast && CallerCC == CalleeCC) { 4899 // Functions containing by val parameters are not supported. 4900 for (unsigned i = 0; i != Ins.size(); i++) { 4901 ISD::ArgFlagsTy Flags = Ins[i].Flags; 4902 if (Flags.isByVal()) return false; 4903 } 4904 4905 // Non-PIC/GOT tail calls are supported. 4906 if (getTargetMachine().getRelocationModel() != Reloc::PIC_) 4907 return true; 4908 4909 // At the moment we can only do local tail calls (in same module, hidden 4910 // or protected) if we are generating PIC. 4911 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) 4912 return G->getGlobal()->hasHiddenVisibility() 4913 || G->getGlobal()->hasProtectedVisibility(); 4914 } 4915 4916 return false; 4917 } 4918 4919 /// isCallCompatibleAddress - Return the immediate to use if the specified 4920 /// 32-bit value is representable in the immediate field of a BxA instruction. 4921 static SDNode *isBLACompatibleAddress(SDValue Op, SelectionDAG &DAG) { 4922 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op); 4923 if (!C) return nullptr; 4924 4925 int Addr = C->getZExtValue(); 4926 if ((Addr & 3) != 0 || // Low 2 bits are implicitly zero. 4927 SignExtend32<26>(Addr) != Addr) 4928 return nullptr; // Top 6 bits have to be sext of immediate. 4929 4930 return DAG 4931 .getConstant( 4932 (int)C->getZExtValue() >> 2, SDLoc(Op), 4933 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout())) 4934 .getNode(); 4935 } 4936 4937 namespace { 4938 4939 struct TailCallArgumentInfo { 4940 SDValue Arg; 4941 SDValue FrameIdxOp; 4942 int FrameIdx = 0; 4943 4944 TailCallArgumentInfo() = default; 4945 }; 4946 4947 } // end anonymous namespace 4948 4949 /// StoreTailCallArgumentsToStackSlot - Stores arguments to their stack slot. 4950 static void StoreTailCallArgumentsToStackSlot( 4951 SelectionDAG &DAG, SDValue Chain, 4952 const SmallVectorImpl<TailCallArgumentInfo> &TailCallArgs, 4953 SmallVectorImpl<SDValue> &MemOpChains, const SDLoc &dl) { 4954 for (unsigned i = 0, e = TailCallArgs.size(); i != e; ++i) { 4955 SDValue Arg = TailCallArgs[i].Arg; 4956 SDValue FIN = TailCallArgs[i].FrameIdxOp; 4957 int FI = TailCallArgs[i].FrameIdx; 4958 // Store relative to framepointer. 4959 MemOpChains.push_back(DAG.getStore( 4960 Chain, dl, Arg, FIN, 4961 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI))); 4962 } 4963 } 4964 4965 /// EmitTailCallStoreFPAndRetAddr - Move the frame pointer and return address to 4966 /// the appropriate stack slot for the tail call optimized function call. 4967 static SDValue EmitTailCallStoreFPAndRetAddr(SelectionDAG &DAG, SDValue Chain, 4968 SDValue OldRetAddr, SDValue OldFP, 4969 int SPDiff, const SDLoc &dl) { 4970 if (SPDiff) { 4971 // Calculate the new stack slot for the return address. 4972 MachineFunction &MF = DAG.getMachineFunction(); 4973 const PPCSubtarget &Subtarget = MF.getSubtarget<PPCSubtarget>(); 4974 const PPCFrameLowering *FL = Subtarget.getFrameLowering(); 4975 bool isPPC64 = Subtarget.isPPC64(); 4976 int SlotSize = isPPC64 ? 8 : 4; 4977 int NewRetAddrLoc = SPDiff + FL->getReturnSaveOffset(); 4978 int NewRetAddr = MF.getFrameInfo().CreateFixedObject(SlotSize, 4979 NewRetAddrLoc, true); 4980 EVT VT = isPPC64 ? MVT::i64 : MVT::i32; 4981 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewRetAddr, VT); 4982 Chain = DAG.getStore(Chain, dl, OldRetAddr, NewRetAddrFrIdx, 4983 MachinePointerInfo::getFixedStack(MF, NewRetAddr)); 4984 } 4985 return Chain; 4986 } 4987 4988 /// CalculateTailCallArgDest - Remember Argument for later processing. Calculate 4989 /// the position of the argument. 4990 static void 4991 CalculateTailCallArgDest(SelectionDAG &DAG, MachineFunction &MF, bool isPPC64, 4992 SDValue Arg, int SPDiff, unsigned ArgOffset, 4993 SmallVectorImpl<TailCallArgumentInfo>& TailCallArguments) { 4994 int Offset = ArgOffset + SPDiff; 4995 uint32_t OpSize = (Arg.getValueSizeInBits() + 7) / 8; 4996 int FI = MF.getFrameInfo().CreateFixedObject(OpSize, Offset, true); 4997 EVT VT = isPPC64 ? MVT::i64 : MVT::i32; 4998 SDValue FIN = DAG.getFrameIndex(FI, VT); 4999 TailCallArgumentInfo Info; 5000 Info.Arg = Arg; 5001 Info.FrameIdxOp = FIN; 5002 Info.FrameIdx = FI; 5003 TailCallArguments.push_back(Info); 5004 } 5005 5006 /// EmitTCFPAndRetAddrLoad - Emit load from frame pointer and return address 5007 /// stack slot. Returns the chain as result and the loaded frame pointers in 5008 /// LROpOut/FPOpout. Used when tail calling. 5009 SDValue PPCTargetLowering::EmitTailCallLoadFPAndRetAddr( 5010 SelectionDAG &DAG, int SPDiff, SDValue Chain, SDValue &LROpOut, 5011 SDValue &FPOpOut, const SDLoc &dl) const { 5012 if (SPDiff) { 5013 // Load the LR and FP stack slot for later adjusting. 5014 EVT VT = Subtarget.isPPC64() ? MVT::i64 : MVT::i32; 5015 LROpOut = getReturnAddrFrameIndex(DAG); 5016 LROpOut = DAG.getLoad(VT, dl, Chain, LROpOut, MachinePointerInfo()); 5017 Chain = SDValue(LROpOut.getNode(), 1); 5018 } 5019 return Chain; 5020 } 5021 5022 /// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified 5023 /// by "Src" to address "Dst" of size "Size". Alignment information is 5024 /// specified by the specific parameter attribute. The copy will be passed as 5025 /// a byval function parameter. 5026 /// Sometimes what we are copying is the end of a larger object, the part that 5027 /// does not fit in registers. 5028 static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst, 5029 SDValue Chain, ISD::ArgFlagsTy Flags, 5030 SelectionDAG &DAG, const SDLoc &dl) { 5031 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32); 5032 return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, 5033 Flags.getNonZeroByValAlign(), false, false, false, 5034 MachinePointerInfo(), MachinePointerInfo()); 5035 } 5036 5037 /// LowerMemOpCallTo - Store the argument to the stack or remember it in case of 5038 /// tail calls. 5039 static void LowerMemOpCallTo( 5040 SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue Arg, 5041 SDValue PtrOff, int SPDiff, unsigned ArgOffset, bool isPPC64, 5042 bool isTailCall, bool isVector, SmallVectorImpl<SDValue> &MemOpChains, 5043 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments, const SDLoc &dl) { 5044 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); 5045 if (!isTailCall) { 5046 if (isVector) { 5047 SDValue StackPtr; 5048 if (isPPC64) 5049 StackPtr = DAG.getRegister(PPC::X1, MVT::i64); 5050 else 5051 StackPtr = DAG.getRegister(PPC::R1, MVT::i32); 5052 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, 5053 DAG.getConstant(ArgOffset, dl, PtrVT)); 5054 } 5055 MemOpChains.push_back( 5056 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo())); 5057 // Calculate and remember argument location. 5058 } else CalculateTailCallArgDest(DAG, MF, isPPC64, Arg, SPDiff, ArgOffset, 5059 TailCallArguments); 5060 } 5061 5062 static void 5063 PrepareTailCall(SelectionDAG &DAG, SDValue &InFlag, SDValue &Chain, 5064 const SDLoc &dl, int SPDiff, unsigned NumBytes, SDValue LROp, 5065 SDValue FPOp, 5066 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments) { 5067 // Emit a sequence of copyto/copyfrom virtual registers for arguments that 5068 // might overwrite each other in case of tail call optimization. 5069 SmallVector<SDValue, 8> MemOpChains2; 5070 // Do not flag preceding copytoreg stuff together with the following stuff. 5071 InFlag = SDValue(); 5072 StoreTailCallArgumentsToStackSlot(DAG, Chain, TailCallArguments, 5073 MemOpChains2, dl); 5074 if (!MemOpChains2.empty()) 5075 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2); 5076 5077 // Store the return address to the appropriate stack slot. 5078 Chain = EmitTailCallStoreFPAndRetAddr(DAG, Chain, LROp, FPOp, SPDiff, dl); 5079 5080 // Emit callseq_end just before tailcall node. 5081 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true), 5082 DAG.getIntPtrConstant(0, dl, true), InFlag, dl); 5083 InFlag = Chain.getValue(1); 5084 } 5085 5086 // Is this global address that of a function that can be called by name? (as 5087 // opposed to something that must hold a descriptor for an indirect call). 5088 static bool isFunctionGlobalAddress(SDValue Callee) { 5089 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 5090 if (Callee.getOpcode() == ISD::GlobalTLSAddress || 5091 Callee.getOpcode() == ISD::TargetGlobalTLSAddress) 5092 return false; 5093 5094 return G->getGlobal()->getValueType()->isFunctionTy(); 5095 } 5096 5097 return false; 5098 } 5099 5100 SDValue PPCTargetLowering::LowerCallResult( 5101 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg, 5102 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 5103 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 5104 SmallVector<CCValAssign, 16> RVLocs; 5105 CCState CCRetInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, 5106 *DAG.getContext()); 5107 5108 CCRetInfo.AnalyzeCallResult( 5109 Ins, (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold) 5110 ? RetCC_PPC_Cold 5111 : RetCC_PPC); 5112 5113 // Copy all of the result registers out of their specified physreg. 5114 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) { 5115 CCValAssign &VA = RVLocs[i]; 5116 assert(VA.isRegLoc() && "Can only return in registers!"); 5117 5118 SDValue Val; 5119 5120 if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) { 5121 SDValue Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, 5122 InFlag); 5123 Chain = Lo.getValue(1); 5124 InFlag = Lo.getValue(2); 5125 VA = RVLocs[++i]; // skip ahead to next loc 5126 SDValue Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, 5127 InFlag); 5128 Chain = Hi.getValue(1); 5129 InFlag = Hi.getValue(2); 5130 if (!Subtarget.isLittleEndian()) 5131 std::swap (Lo, Hi); 5132 Val = DAG.getNode(PPCISD::BUILD_SPE64, dl, MVT::f64, Lo, Hi); 5133 } else { 5134 Val = DAG.getCopyFromReg(Chain, dl, 5135 VA.getLocReg(), VA.getLocVT(), InFlag); 5136 Chain = Val.getValue(1); 5137 InFlag = Val.getValue(2); 5138 } 5139 5140 switch (VA.getLocInfo()) { 5141 default: llvm_unreachable("Unknown loc info!"); 5142 case CCValAssign::Full: break; 5143 case CCValAssign::AExt: 5144 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); 5145 break; 5146 case CCValAssign::ZExt: 5147 Val = DAG.getNode(ISD::AssertZext, dl, VA.getLocVT(), Val, 5148 DAG.getValueType(VA.getValVT())); 5149 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); 5150 break; 5151 case CCValAssign::SExt: 5152 Val = DAG.getNode(ISD::AssertSext, dl, VA.getLocVT(), Val, 5153 DAG.getValueType(VA.getValVT())); 5154 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); 5155 break; 5156 } 5157 5158 InVals.push_back(Val); 5159 } 5160 5161 return Chain; 5162 } 5163 5164 static bool isIndirectCall(const SDValue &Callee, SelectionDAG &DAG, 5165 const PPCSubtarget &Subtarget, bool isPatchPoint) { 5166 // PatchPoint calls are not indirect. 5167 if (isPatchPoint) 5168 return false; 5169 5170 if (isFunctionGlobalAddress(Callee) || isa<ExternalSymbolSDNode>(Callee)) 5171 return false; 5172 5173 // Darwin, and 32-bit ELF can use a BLA. The descriptor based ABIs can not 5174 // becuase the immediate function pointer points to a descriptor instead of 5175 // a function entry point. The ELFv2 ABI cannot use a BLA because the function 5176 // pointer immediate points to the global entry point, while the BLA would 5177 // need to jump to the local entry point (see rL211174). 5178 if (!Subtarget.usesFunctionDescriptors() && !Subtarget.isELFv2ABI() && 5179 isBLACompatibleAddress(Callee, DAG)) 5180 return false; 5181 5182 return true; 5183 } 5184 5185 // AIX and 64-bit ELF ABIs w/o PCRel require a TOC save/restore around calls. 5186 static inline bool isTOCSaveRestoreRequired(const PPCSubtarget &Subtarget) { 5187 return Subtarget.isAIXABI() || 5188 (Subtarget.is64BitELFABI() && !Subtarget.isUsingPCRelativeCalls()); 5189 } 5190 5191 static unsigned getCallOpcode(PPCTargetLowering::CallFlags CFlags, 5192 const Function &Caller, const SDValue &Callee, 5193 const PPCSubtarget &Subtarget, 5194 const TargetMachine &TM, 5195 bool IsStrictFPCall = false) { 5196 if (CFlags.IsTailCall) 5197 return PPCISD::TC_RETURN; 5198 5199 unsigned RetOpc = 0; 5200 // This is a call through a function pointer. 5201 if (CFlags.IsIndirect) { 5202 // AIX and the 64-bit ELF ABIs need to maintain the TOC pointer accross 5203 // indirect calls. The save of the caller's TOC pointer to the stack will be 5204 // inserted into the DAG as part of call lowering. The restore of the TOC 5205 // pointer is modeled by using a pseudo instruction for the call opcode that 5206 // represents the 2 instruction sequence of an indirect branch and link, 5207 // immediately followed by a load of the TOC pointer from the the stack save 5208 // slot into gpr2. For 64-bit ELFv2 ABI with PCRel, do not restore the TOC 5209 // as it is not saved or used. 5210 RetOpc = isTOCSaveRestoreRequired(Subtarget) ? PPCISD::BCTRL_LOAD_TOC 5211 : PPCISD::BCTRL; 5212 } else if (Subtarget.isUsingPCRelativeCalls()) { 5213 assert(Subtarget.is64BitELFABI() && "PC Relative is only on ELF ABI."); 5214 RetOpc = PPCISD::CALL_NOTOC; 5215 } else if (Subtarget.isAIXABI() || Subtarget.is64BitELFABI()) 5216 // The ABIs that maintain a TOC pointer accross calls need to have a nop 5217 // immediately following the call instruction if the caller and callee may 5218 // have different TOC bases. At link time if the linker determines the calls 5219 // may not share a TOC base, the call is redirected to a trampoline inserted 5220 // by the linker. The trampoline will (among other things) save the callers 5221 // TOC pointer at an ABI designated offset in the linkage area and the 5222 // linker will rewrite the nop to be a load of the TOC pointer from the 5223 // linkage area into gpr2. 5224 RetOpc = callsShareTOCBase(&Caller, Callee, TM) ? PPCISD::CALL 5225 : PPCISD::CALL_NOP; 5226 else 5227 RetOpc = PPCISD::CALL; 5228 if (IsStrictFPCall) { 5229 switch (RetOpc) { 5230 default: 5231 llvm_unreachable("Unknown call opcode"); 5232 case PPCISD::BCTRL_LOAD_TOC: 5233 RetOpc = PPCISD::BCTRL_LOAD_TOC_RM; 5234 break; 5235 case PPCISD::BCTRL: 5236 RetOpc = PPCISD::BCTRL_RM; 5237 break; 5238 case PPCISD::CALL_NOTOC: 5239 RetOpc = PPCISD::CALL_NOTOC_RM; 5240 break; 5241 case PPCISD::CALL: 5242 RetOpc = PPCISD::CALL_RM; 5243 break; 5244 case PPCISD::CALL_NOP: 5245 RetOpc = PPCISD::CALL_NOP_RM; 5246 break; 5247 } 5248 } 5249 return RetOpc; 5250 } 5251 5252 static SDValue transformCallee(const SDValue &Callee, SelectionDAG &DAG, 5253 const SDLoc &dl, const PPCSubtarget &Subtarget) { 5254 if (!Subtarget.usesFunctionDescriptors() && !Subtarget.isELFv2ABI()) 5255 if (SDNode *Dest = isBLACompatibleAddress(Callee, DAG)) 5256 return SDValue(Dest, 0); 5257 5258 // Returns true if the callee is local, and false otherwise. 5259 auto isLocalCallee = [&]() { 5260 const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee); 5261 const Module *Mod = DAG.getMachineFunction().getFunction().getParent(); 5262 const GlobalValue *GV = G ? G->getGlobal() : nullptr; 5263 5264 return DAG.getTarget().shouldAssumeDSOLocal(*Mod, GV) && 5265 !isa_and_nonnull<GlobalIFunc>(GV); 5266 }; 5267 5268 // The PLT is only used in 32-bit ELF PIC mode. Attempting to use the PLT in 5269 // a static relocation model causes some versions of GNU LD (2.17.50, at 5270 // least) to force BSS-PLT, instead of secure-PLT, even if all objects are 5271 // built with secure-PLT. 5272 bool UsePlt = 5273 Subtarget.is32BitELFABI() && !isLocalCallee() && 5274 Subtarget.getTargetMachine().getRelocationModel() == Reloc::PIC_; 5275 5276 const auto getAIXFuncEntryPointSymbolSDNode = [&](const GlobalValue *GV) { 5277 const TargetMachine &TM = Subtarget.getTargetMachine(); 5278 const TargetLoweringObjectFile *TLOF = TM.getObjFileLowering(); 5279 MCSymbolXCOFF *S = 5280 cast<MCSymbolXCOFF>(TLOF->getFunctionEntryPointSymbol(GV, TM)); 5281 5282 MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); 5283 return DAG.getMCSymbol(S, PtrVT); 5284 }; 5285 5286 if (isFunctionGlobalAddress(Callee)) { 5287 const GlobalValue *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal(); 5288 5289 if (Subtarget.isAIXABI()) { 5290 assert(!isa<GlobalIFunc>(GV) && "IFunc is not supported on AIX."); 5291 return getAIXFuncEntryPointSymbolSDNode(GV); 5292 } 5293 return DAG.getTargetGlobalAddress(GV, dl, Callee.getValueType(), 0, 5294 UsePlt ? PPCII::MO_PLT : 0); 5295 } 5296 5297 if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) { 5298 const char *SymName = S->getSymbol(); 5299 if (Subtarget.isAIXABI()) { 5300 // If there exists a user-declared function whose name is the same as the 5301 // ExternalSymbol's, then we pick up the user-declared version. 5302 const Module *Mod = DAG.getMachineFunction().getFunction().getParent(); 5303 if (const Function *F = 5304 dyn_cast_or_null<Function>(Mod->getNamedValue(SymName))) 5305 return getAIXFuncEntryPointSymbolSDNode(F); 5306 5307 // On AIX, direct function calls reference the symbol for the function's 5308 // entry point, which is named by prepending a "." before the function's 5309 // C-linkage name. A Qualname is returned here because an external 5310 // function entry point is a csect with XTY_ER property. 5311 const auto getExternalFunctionEntryPointSymbol = [&](StringRef SymName) { 5312 auto &Context = DAG.getMachineFunction().getMMI().getContext(); 5313 MCSectionXCOFF *Sec = Context.getXCOFFSection( 5314 (Twine(".") + Twine(SymName)).str(), SectionKind::getMetadata(), 5315 XCOFF::CsectProperties(XCOFF::XMC_PR, XCOFF::XTY_ER)); 5316 return Sec->getQualNameSymbol(); 5317 }; 5318 5319 SymName = getExternalFunctionEntryPointSymbol(SymName)->getName().data(); 5320 } 5321 return DAG.getTargetExternalSymbol(SymName, Callee.getValueType(), 5322 UsePlt ? PPCII::MO_PLT : 0); 5323 } 5324 5325 // No transformation needed. 5326 assert(Callee.getNode() && "What no callee?"); 5327 return Callee; 5328 } 5329 5330 static SDValue getOutputChainFromCallSeq(SDValue CallSeqStart) { 5331 assert(CallSeqStart.getOpcode() == ISD::CALLSEQ_START && 5332 "Expected a CALLSEQ_STARTSDNode."); 5333 5334 // The last operand is the chain, except when the node has glue. If the node 5335 // has glue, then the last operand is the glue, and the chain is the second 5336 // last operand. 5337 SDValue LastValue = CallSeqStart.getValue(CallSeqStart->getNumValues() - 1); 5338 if (LastValue.getValueType() != MVT::Glue) 5339 return LastValue; 5340 5341 return CallSeqStart.getValue(CallSeqStart->getNumValues() - 2); 5342 } 5343 5344 // Creates the node that moves a functions address into the count register 5345 // to prepare for an indirect call instruction. 5346 static void prepareIndirectCall(SelectionDAG &DAG, SDValue &Callee, 5347 SDValue &Glue, SDValue &Chain, 5348 const SDLoc &dl) { 5349 SDValue MTCTROps[] = {Chain, Callee, Glue}; 5350 EVT ReturnTypes[] = {MVT::Other, MVT::Glue}; 5351 Chain = DAG.getNode(PPCISD::MTCTR, dl, makeArrayRef(ReturnTypes, 2), 5352 makeArrayRef(MTCTROps, Glue.getNode() ? 3 : 2)); 5353 // The glue is the second value produced. 5354 Glue = Chain.getValue(1); 5355 } 5356 5357 static void prepareDescriptorIndirectCall(SelectionDAG &DAG, SDValue &Callee, 5358 SDValue &Glue, SDValue &Chain, 5359 SDValue CallSeqStart, 5360 const CallBase *CB, const SDLoc &dl, 5361 bool hasNest, 5362 const PPCSubtarget &Subtarget) { 5363 // Function pointers in the 64-bit SVR4 ABI do not point to the function 5364 // entry point, but to the function descriptor (the function entry point 5365 // address is part of the function descriptor though). 5366 // The function descriptor is a three doubleword structure with the 5367 // following fields: function entry point, TOC base address and 5368 // environment pointer. 5369 // Thus for a call through a function pointer, the following actions need 5370 // to be performed: 5371 // 1. Save the TOC of the caller in the TOC save area of its stack 5372 // frame (this is done in LowerCall_Darwin() or LowerCall_64SVR4()). 5373 // 2. Load the address of the function entry point from the function 5374 // descriptor. 5375 // 3. Load the TOC of the callee from the function descriptor into r2. 5376 // 4. Load the environment pointer from the function descriptor into 5377 // r11. 5378 // 5. Branch to the function entry point address. 5379 // 6. On return of the callee, the TOC of the caller needs to be 5380 // restored (this is done in FinishCall()). 5381 // 5382 // The loads are scheduled at the beginning of the call sequence, and the 5383 // register copies are flagged together to ensure that no other 5384 // operations can be scheduled in between. E.g. without flagging the 5385 // copies together, a TOC access in the caller could be scheduled between 5386 // the assignment of the callee TOC and the branch to the callee, which leads 5387 // to incorrect code. 5388 5389 // Start by loading the function address from the descriptor. 5390 SDValue LDChain = getOutputChainFromCallSeq(CallSeqStart); 5391 auto MMOFlags = Subtarget.hasInvariantFunctionDescriptors() 5392 ? (MachineMemOperand::MODereferenceable | 5393 MachineMemOperand::MOInvariant) 5394 : MachineMemOperand::MONone; 5395 5396 MachinePointerInfo MPI(CB ? CB->getCalledOperand() : nullptr); 5397 5398 // Registers used in building the DAG. 5399 const MCRegister EnvPtrReg = Subtarget.getEnvironmentPointerRegister(); 5400 const MCRegister TOCReg = Subtarget.getTOCPointerRegister(); 5401 5402 // Offsets of descriptor members. 5403 const unsigned TOCAnchorOffset = Subtarget.descriptorTOCAnchorOffset(); 5404 const unsigned EnvPtrOffset = Subtarget.descriptorEnvironmentPointerOffset(); 5405 5406 const MVT RegVT = Subtarget.isPPC64() ? MVT::i64 : MVT::i32; 5407 const unsigned Alignment = Subtarget.isPPC64() ? 8 : 4; 5408 5409 // One load for the functions entry point address. 5410 SDValue LoadFuncPtr = DAG.getLoad(RegVT, dl, LDChain, Callee, MPI, 5411 Alignment, MMOFlags); 5412 5413 // One for loading the TOC anchor for the module that contains the called 5414 // function. 5415 SDValue TOCOff = DAG.getIntPtrConstant(TOCAnchorOffset, dl); 5416 SDValue AddTOC = DAG.getNode(ISD::ADD, dl, RegVT, Callee, TOCOff); 5417 SDValue TOCPtr = 5418 DAG.getLoad(RegVT, dl, LDChain, AddTOC, 5419 MPI.getWithOffset(TOCAnchorOffset), Alignment, MMOFlags); 5420 5421 // One for loading the environment pointer. 5422 SDValue PtrOff = DAG.getIntPtrConstant(EnvPtrOffset, dl); 5423 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, RegVT, Callee, PtrOff); 5424 SDValue LoadEnvPtr = 5425 DAG.getLoad(RegVT, dl, LDChain, AddPtr, 5426 MPI.getWithOffset(EnvPtrOffset), Alignment, MMOFlags); 5427 5428 5429 // Then copy the newly loaded TOC anchor to the TOC pointer. 5430 SDValue TOCVal = DAG.getCopyToReg(Chain, dl, TOCReg, TOCPtr, Glue); 5431 Chain = TOCVal.getValue(0); 5432 Glue = TOCVal.getValue(1); 5433 5434 // If the function call has an explicit 'nest' parameter, it takes the 5435 // place of the environment pointer. 5436 assert((!hasNest || !Subtarget.isAIXABI()) && 5437 "Nest parameter is not supported on AIX."); 5438 if (!hasNest) { 5439 SDValue EnvVal = DAG.getCopyToReg(Chain, dl, EnvPtrReg, LoadEnvPtr, Glue); 5440 Chain = EnvVal.getValue(0); 5441 Glue = EnvVal.getValue(1); 5442 } 5443 5444 // The rest of the indirect call sequence is the same as the non-descriptor 5445 // DAG. 5446 prepareIndirectCall(DAG, LoadFuncPtr, Glue, Chain, dl); 5447 } 5448 5449 static void 5450 buildCallOperands(SmallVectorImpl<SDValue> &Ops, 5451 PPCTargetLowering::CallFlags CFlags, const SDLoc &dl, 5452 SelectionDAG &DAG, 5453 SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass, 5454 SDValue Glue, SDValue Chain, SDValue &Callee, int SPDiff, 5455 const PPCSubtarget &Subtarget) { 5456 const bool IsPPC64 = Subtarget.isPPC64(); 5457 // MVT for a general purpose register. 5458 const MVT RegVT = IsPPC64 ? MVT::i64 : MVT::i32; 5459 5460 // First operand is always the chain. 5461 Ops.push_back(Chain); 5462 5463 // If it's a direct call pass the callee as the second operand. 5464 if (!CFlags.IsIndirect) 5465 Ops.push_back(Callee); 5466 else { 5467 assert(!CFlags.IsPatchPoint && "Patch point calls are not indirect."); 5468 5469 // For the TOC based ABIs, we have saved the TOC pointer to the linkage area 5470 // on the stack (this would have been done in `LowerCall_64SVR4` or 5471 // `LowerCall_AIX`). The call instruction is a pseudo instruction that 5472 // represents both the indirect branch and a load that restores the TOC 5473 // pointer from the linkage area. The operand for the TOC restore is an add 5474 // of the TOC save offset to the stack pointer. This must be the second 5475 // operand: after the chain input but before any other variadic arguments. 5476 // For 64-bit ELFv2 ABI with PCRel, do not restore the TOC as it is not 5477 // saved or used. 5478 if (isTOCSaveRestoreRequired(Subtarget)) { 5479 const MCRegister StackPtrReg = Subtarget.getStackPointerRegister(); 5480 5481 SDValue StackPtr = DAG.getRegister(StackPtrReg, RegVT); 5482 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset(); 5483 SDValue TOCOff = DAG.getIntPtrConstant(TOCSaveOffset, dl); 5484 SDValue AddTOC = DAG.getNode(ISD::ADD, dl, RegVT, StackPtr, TOCOff); 5485 Ops.push_back(AddTOC); 5486 } 5487 5488 // Add the register used for the environment pointer. 5489 if (Subtarget.usesFunctionDescriptors() && !CFlags.HasNest) 5490 Ops.push_back(DAG.getRegister(Subtarget.getEnvironmentPointerRegister(), 5491 RegVT)); 5492 5493 5494 // Add CTR register as callee so a bctr can be emitted later. 5495 if (CFlags.IsTailCall) 5496 Ops.push_back(DAG.getRegister(IsPPC64 ? PPC::CTR8 : PPC::CTR, RegVT)); 5497 } 5498 5499 // If this is a tail call add stack pointer delta. 5500 if (CFlags.IsTailCall) 5501 Ops.push_back(DAG.getConstant(SPDiff, dl, MVT::i32)); 5502 5503 // Add argument registers to the end of the list so that they are known live 5504 // into the call. 5505 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) 5506 Ops.push_back(DAG.getRegister(RegsToPass[i].first, 5507 RegsToPass[i].second.getValueType())); 5508 5509 // We cannot add R2/X2 as an operand here for PATCHPOINT, because there is 5510 // no way to mark dependencies as implicit here. 5511 // We will add the R2/X2 dependency in EmitInstrWithCustomInserter. 5512 if ((Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) && 5513 !CFlags.IsPatchPoint && !Subtarget.isUsingPCRelativeCalls()) 5514 Ops.push_back(DAG.getRegister(Subtarget.getTOCPointerRegister(), RegVT)); 5515 5516 // Add implicit use of CR bit 6 for 32-bit SVR4 vararg calls 5517 if (CFlags.IsVarArg && Subtarget.is32BitELFABI()) 5518 Ops.push_back(DAG.getRegister(PPC::CR1EQ, MVT::i32)); 5519 5520 // Add a register mask operand representing the call-preserved registers. 5521 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); 5522 const uint32_t *Mask = 5523 TRI->getCallPreservedMask(DAG.getMachineFunction(), CFlags.CallConv); 5524 assert(Mask && "Missing call preserved mask for calling convention"); 5525 Ops.push_back(DAG.getRegisterMask(Mask)); 5526 5527 // If the glue is valid, it is the last operand. 5528 if (Glue.getNode()) 5529 Ops.push_back(Glue); 5530 } 5531 5532 SDValue PPCTargetLowering::FinishCall( 5533 CallFlags CFlags, const SDLoc &dl, SelectionDAG &DAG, 5534 SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass, SDValue Glue, 5535 SDValue Chain, SDValue CallSeqStart, SDValue &Callee, int SPDiff, 5536 unsigned NumBytes, const SmallVectorImpl<ISD::InputArg> &Ins, 5537 SmallVectorImpl<SDValue> &InVals, const CallBase *CB) const { 5538 5539 if ((Subtarget.is64BitELFABI() && !Subtarget.isUsingPCRelativeCalls()) || 5540 Subtarget.isAIXABI()) 5541 setUsesTOCBasePtr(DAG); 5542 5543 unsigned CallOpc = 5544 getCallOpcode(CFlags, DAG.getMachineFunction().getFunction(), Callee, 5545 Subtarget, DAG.getTarget(), CB ? CB->isStrictFP() : false); 5546 5547 if (!CFlags.IsIndirect) 5548 Callee = transformCallee(Callee, DAG, dl, Subtarget); 5549 else if (Subtarget.usesFunctionDescriptors()) 5550 prepareDescriptorIndirectCall(DAG, Callee, Glue, Chain, CallSeqStart, CB, 5551 dl, CFlags.HasNest, Subtarget); 5552 else 5553 prepareIndirectCall(DAG, Callee, Glue, Chain, dl); 5554 5555 // Build the operand list for the call instruction. 5556 SmallVector<SDValue, 8> Ops; 5557 buildCallOperands(Ops, CFlags, dl, DAG, RegsToPass, Glue, Chain, Callee, 5558 SPDiff, Subtarget); 5559 5560 // Emit tail call. 5561 if (CFlags.IsTailCall) { 5562 // Indirect tail call when using PC Relative calls do not have the same 5563 // constraints. 5564 assert(((Callee.getOpcode() == ISD::Register && 5565 cast<RegisterSDNode>(Callee)->getReg() == PPC::CTR) || 5566 Callee.getOpcode() == ISD::TargetExternalSymbol || 5567 Callee.getOpcode() == ISD::TargetGlobalAddress || 5568 isa<ConstantSDNode>(Callee) || 5569 (CFlags.IsIndirect && Subtarget.isUsingPCRelativeCalls())) && 5570 "Expecting a global address, external symbol, absolute value, " 5571 "register or an indirect tail call when PC Relative calls are " 5572 "used."); 5573 // PC Relative calls also use TC_RETURN as the way to mark tail calls. 5574 assert(CallOpc == PPCISD::TC_RETURN && 5575 "Unexpected call opcode for a tail call."); 5576 DAG.getMachineFunction().getFrameInfo().setHasTailCall(); 5577 return DAG.getNode(CallOpc, dl, MVT::Other, Ops); 5578 } 5579 5580 std::array<EVT, 2> ReturnTypes = {{MVT::Other, MVT::Glue}}; 5581 Chain = DAG.getNode(CallOpc, dl, ReturnTypes, Ops); 5582 DAG.addNoMergeSiteInfo(Chain.getNode(), CFlags.NoMerge); 5583 Glue = Chain.getValue(1); 5584 5585 // When performing tail call optimization the callee pops its arguments off 5586 // the stack. Account for this here so these bytes can be pushed back on in 5587 // PPCFrameLowering::eliminateCallFramePseudoInstr. 5588 int BytesCalleePops = (CFlags.CallConv == CallingConv::Fast && 5589 getTargetMachine().Options.GuaranteedTailCallOpt) 5590 ? NumBytes 5591 : 0; 5592 5593 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true), 5594 DAG.getIntPtrConstant(BytesCalleePops, dl, true), 5595 Glue, dl); 5596 Glue = Chain.getValue(1); 5597 5598 return LowerCallResult(Chain, Glue, CFlags.CallConv, CFlags.IsVarArg, Ins, dl, 5599 DAG, InVals); 5600 } 5601 5602 SDValue 5603 PPCTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, 5604 SmallVectorImpl<SDValue> &InVals) const { 5605 SelectionDAG &DAG = CLI.DAG; 5606 SDLoc &dl = CLI.DL; 5607 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; 5608 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; 5609 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; 5610 SDValue Chain = CLI.Chain; 5611 SDValue Callee = CLI.Callee; 5612 bool &isTailCall = CLI.IsTailCall; 5613 CallingConv::ID CallConv = CLI.CallConv; 5614 bool isVarArg = CLI.IsVarArg; 5615 bool isPatchPoint = CLI.IsPatchPoint; 5616 const CallBase *CB = CLI.CB; 5617 5618 if (isTailCall) { 5619 if (Subtarget.useLongCalls() && !(CB && CB->isMustTailCall())) 5620 isTailCall = false; 5621 else if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) 5622 isTailCall = IsEligibleForTailCallOptimization_64SVR4( 5623 Callee, CallConv, CB, isVarArg, Outs, Ins, DAG); 5624 else 5625 isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv, isVarArg, 5626 Ins, DAG); 5627 if (isTailCall) { 5628 ++NumTailCalls; 5629 if (!getTargetMachine().Options.GuaranteedTailCallOpt) 5630 ++NumSiblingCalls; 5631 5632 // PC Relative calls no longer guarantee that the callee is a Global 5633 // Address Node. The callee could be an indirect tail call in which 5634 // case the SDValue for the callee could be a load (to load the address 5635 // of a function pointer) or it may be a register copy (to move the 5636 // address of the callee from a function parameter into a virtual 5637 // register). It may also be an ExternalSymbolSDNode (ex memcopy). 5638 assert((Subtarget.isUsingPCRelativeCalls() || 5639 isa<GlobalAddressSDNode>(Callee)) && 5640 "Callee should be an llvm::Function object."); 5641 5642 LLVM_DEBUG(dbgs() << "TCO caller: " << DAG.getMachineFunction().getName() 5643 << "\nTCO callee: "); 5644 LLVM_DEBUG(Callee.dump()); 5645 } 5646 } 5647 5648 if (!isTailCall && CB && CB->isMustTailCall()) 5649 report_fatal_error("failed to perform tail call elimination on a call " 5650 "site marked musttail"); 5651 5652 // When long calls (i.e. indirect calls) are always used, calls are always 5653 // made via function pointer. If we have a function name, first translate it 5654 // into a pointer. 5655 if (Subtarget.useLongCalls() && isa<GlobalAddressSDNode>(Callee) && 5656 !isTailCall) 5657 Callee = LowerGlobalAddress(Callee, DAG); 5658 5659 CallFlags CFlags( 5660 CallConv, isTailCall, isVarArg, isPatchPoint, 5661 isIndirectCall(Callee, DAG, Subtarget, isPatchPoint), 5662 // hasNest 5663 Subtarget.is64BitELFABI() && 5664 any_of(Outs, [](ISD::OutputArg Arg) { return Arg.Flags.isNest(); }), 5665 CLI.NoMerge); 5666 5667 if (Subtarget.isAIXABI()) 5668 return LowerCall_AIX(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG, 5669 InVals, CB); 5670 5671 assert(Subtarget.isSVR4ABI()); 5672 if (Subtarget.isPPC64()) 5673 return LowerCall_64SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG, 5674 InVals, CB); 5675 return LowerCall_32SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG, 5676 InVals, CB); 5677 } 5678 5679 SDValue PPCTargetLowering::LowerCall_32SVR4( 5680 SDValue Chain, SDValue Callee, CallFlags CFlags, 5681 const SmallVectorImpl<ISD::OutputArg> &Outs, 5682 const SmallVectorImpl<SDValue> &OutVals, 5683 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 5684 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, 5685 const CallBase *CB) const { 5686 // See PPCTargetLowering::LowerFormalArguments_32SVR4() for a description 5687 // of the 32-bit SVR4 ABI stack frame layout. 5688 5689 const CallingConv::ID CallConv = CFlags.CallConv; 5690 const bool IsVarArg = CFlags.IsVarArg; 5691 const bool IsTailCall = CFlags.IsTailCall; 5692 5693 assert((CallConv == CallingConv::C || 5694 CallConv == CallingConv::Cold || 5695 CallConv == CallingConv::Fast) && "Unknown calling convention!"); 5696 5697 const Align PtrAlign(4); 5698 5699 MachineFunction &MF = DAG.getMachineFunction(); 5700 5701 // Mark this function as potentially containing a function that contains a 5702 // tail call. As a consequence the frame pointer will be used for dynamicalloc 5703 // and restoring the callers stack pointer in this functions epilog. This is 5704 // done because by tail calling the called function might overwrite the value 5705 // in this function's (MF) stack pointer stack slot 0(SP). 5706 if (getTargetMachine().Options.GuaranteedTailCallOpt && 5707 CallConv == CallingConv::Fast) 5708 MF.getInfo<PPCFunctionInfo>()->setHasFastCall(); 5709 5710 // Count how many bytes are to be pushed on the stack, including the linkage 5711 // area, parameter list area and the part of the local variable space which 5712 // contains copies of aggregates which are passed by value. 5713 5714 // Assign locations to all of the outgoing arguments. 5715 SmallVector<CCValAssign, 16> ArgLocs; 5716 PPCCCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); 5717 5718 // Reserve space for the linkage area on the stack. 5719 CCInfo.AllocateStack(Subtarget.getFrameLowering()->getLinkageSize(), 5720 PtrAlign); 5721 if (useSoftFloat()) 5722 CCInfo.PreAnalyzeCallOperands(Outs); 5723 5724 if (IsVarArg) { 5725 // Handle fixed and variable vector arguments differently. 5726 // Fixed vector arguments go into registers as long as registers are 5727 // available. Variable vector arguments always go into memory. 5728 unsigned NumArgs = Outs.size(); 5729 5730 for (unsigned i = 0; i != NumArgs; ++i) { 5731 MVT ArgVT = Outs[i].VT; 5732 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; 5733 bool Result; 5734 5735 if (Outs[i].IsFixed) { 5736 Result = CC_PPC32_SVR4(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, 5737 CCInfo); 5738 } else { 5739 Result = CC_PPC32_SVR4_VarArg(i, ArgVT, ArgVT, CCValAssign::Full, 5740 ArgFlags, CCInfo); 5741 } 5742 5743 if (Result) { 5744 #ifndef NDEBUG 5745 errs() << "Call operand #" << i << " has unhandled type " 5746 << EVT(ArgVT).getEVTString() << "\n"; 5747 #endif 5748 llvm_unreachable(nullptr); 5749 } 5750 } 5751 } else { 5752 // All arguments are treated the same. 5753 CCInfo.AnalyzeCallOperands(Outs, CC_PPC32_SVR4); 5754 } 5755 CCInfo.clearWasPPCF128(); 5756 5757 // Assign locations to all of the outgoing aggregate by value arguments. 5758 SmallVector<CCValAssign, 16> ByValArgLocs; 5759 CCState CCByValInfo(CallConv, IsVarArg, MF, ByValArgLocs, *DAG.getContext()); 5760 5761 // Reserve stack space for the allocations in CCInfo. 5762 CCByValInfo.AllocateStack(CCInfo.getNextStackOffset(), PtrAlign); 5763 5764 CCByValInfo.AnalyzeCallOperands(Outs, CC_PPC32_SVR4_ByVal); 5765 5766 // Size of the linkage area, parameter list area and the part of the local 5767 // space variable where copies of aggregates which are passed by value are 5768 // stored. 5769 unsigned NumBytes = CCByValInfo.getNextStackOffset(); 5770 5771 // Calculate by how many bytes the stack has to be adjusted in case of tail 5772 // call optimization. 5773 int SPDiff = CalculateTailCallSPDiff(DAG, IsTailCall, NumBytes); 5774 5775 // Adjust the stack pointer for the new arguments... 5776 // These operations are automatically eliminated by the prolog/epilog pass 5777 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); 5778 SDValue CallSeqStart = Chain; 5779 5780 // Load the return address and frame pointer so it can be moved somewhere else 5781 // later. 5782 SDValue LROp, FPOp; 5783 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, dl); 5784 5785 // Set up a copy of the stack pointer for use loading and storing any 5786 // arguments that may not fit in the registers available for argument 5787 // passing. 5788 SDValue StackPtr = DAG.getRegister(PPC::R1, MVT::i32); 5789 5790 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 5791 SmallVector<TailCallArgumentInfo, 8> TailCallArguments; 5792 SmallVector<SDValue, 8> MemOpChains; 5793 5794 bool seenFloatArg = false; 5795 // Walk the register/memloc assignments, inserting copies/loads. 5796 // i - Tracks the index into the list of registers allocated for the call 5797 // RealArgIdx - Tracks the index into the list of actual function arguments 5798 // j - Tracks the index into the list of byval arguments 5799 for (unsigned i = 0, RealArgIdx = 0, j = 0, e = ArgLocs.size(); 5800 i != e; 5801 ++i, ++RealArgIdx) { 5802 CCValAssign &VA = ArgLocs[i]; 5803 SDValue Arg = OutVals[RealArgIdx]; 5804 ISD::ArgFlagsTy Flags = Outs[RealArgIdx].Flags; 5805 5806 if (Flags.isByVal()) { 5807 // Argument is an aggregate which is passed by value, thus we need to 5808 // create a copy of it in the local variable space of the current stack 5809 // frame (which is the stack frame of the caller) and pass the address of 5810 // this copy to the callee. 5811 assert((j < ByValArgLocs.size()) && "Index out of bounds!"); 5812 CCValAssign &ByValVA = ByValArgLocs[j++]; 5813 assert((VA.getValNo() == ByValVA.getValNo()) && "ValNo mismatch!"); 5814 5815 // Memory reserved in the local variable space of the callers stack frame. 5816 unsigned LocMemOffset = ByValVA.getLocMemOffset(); 5817 5818 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl); 5819 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(MF.getDataLayout()), 5820 StackPtr, PtrOff); 5821 5822 // Create a copy of the argument in the local area of the current 5823 // stack frame. 5824 SDValue MemcpyCall = 5825 CreateCopyOfByValArgument(Arg, PtrOff, 5826 CallSeqStart.getNode()->getOperand(0), 5827 Flags, DAG, dl); 5828 5829 // This must go outside the CALLSEQ_START..END. 5830 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall, NumBytes, 0, 5831 SDLoc(MemcpyCall)); 5832 DAG.ReplaceAllUsesWith(CallSeqStart.getNode(), 5833 NewCallSeqStart.getNode()); 5834 Chain = CallSeqStart = NewCallSeqStart; 5835 5836 // Pass the address of the aggregate copy on the stack either in a 5837 // physical register or in the parameter list area of the current stack 5838 // frame to the callee. 5839 Arg = PtrOff; 5840 } 5841 5842 // When useCRBits() is true, there can be i1 arguments. 5843 // It is because getRegisterType(MVT::i1) => MVT::i1, 5844 // and for other integer types getRegisterType() => MVT::i32. 5845 // Extend i1 and ensure callee will get i32. 5846 if (Arg.getValueType() == MVT::i1) 5847 Arg = DAG.getNode(Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, 5848 dl, MVT::i32, Arg); 5849 5850 if (VA.isRegLoc()) { 5851 seenFloatArg |= VA.getLocVT().isFloatingPoint(); 5852 // Put argument in a physical register. 5853 if (Subtarget.hasSPE() && Arg.getValueType() == MVT::f64) { 5854 bool IsLE = Subtarget.isLittleEndian(); 5855 SDValue SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg, 5856 DAG.getIntPtrConstant(IsLE ? 0 : 1, dl)); 5857 RegsToPass.push_back(std::make_pair(VA.getLocReg(), SVal.getValue(0))); 5858 SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg, 5859 DAG.getIntPtrConstant(IsLE ? 1 : 0, dl)); 5860 RegsToPass.push_back(std::make_pair(ArgLocs[++i].getLocReg(), 5861 SVal.getValue(0))); 5862 } else 5863 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 5864 } else { 5865 // Put argument in the parameter list area of the current stack frame. 5866 assert(VA.isMemLoc()); 5867 unsigned LocMemOffset = VA.getLocMemOffset(); 5868 5869 if (!IsTailCall) { 5870 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl); 5871 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(MF.getDataLayout()), 5872 StackPtr, PtrOff); 5873 5874 MemOpChains.push_back( 5875 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo())); 5876 } else { 5877 // Calculate and remember argument location. 5878 CalculateTailCallArgDest(DAG, MF, false, Arg, SPDiff, LocMemOffset, 5879 TailCallArguments); 5880 } 5881 } 5882 } 5883 5884 if (!MemOpChains.empty()) 5885 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); 5886 5887 // Build a sequence of copy-to-reg nodes chained together with token chain 5888 // and flag operands which copy the outgoing args into the appropriate regs. 5889 SDValue InFlag; 5890 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 5891 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 5892 RegsToPass[i].second, InFlag); 5893 InFlag = Chain.getValue(1); 5894 } 5895 5896 // Set CR bit 6 to true if this is a vararg call with floating args passed in 5897 // registers. 5898 if (IsVarArg) { 5899 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue); 5900 SDValue Ops[] = { Chain, InFlag }; 5901 5902 Chain = DAG.getNode(seenFloatArg ? PPCISD::CR6SET : PPCISD::CR6UNSET, 5903 dl, VTs, makeArrayRef(Ops, InFlag.getNode() ? 2 : 1)); 5904 5905 InFlag = Chain.getValue(1); 5906 } 5907 5908 if (IsTailCall) 5909 PrepareTailCall(DAG, InFlag, Chain, dl, SPDiff, NumBytes, LROp, FPOp, 5910 TailCallArguments); 5911 5912 return FinishCall(CFlags, dl, DAG, RegsToPass, InFlag, Chain, CallSeqStart, 5913 Callee, SPDiff, NumBytes, Ins, InVals, CB); 5914 } 5915 5916 // Copy an argument into memory, being careful to do this outside the 5917 // call sequence for the call to which the argument belongs. 5918 SDValue PPCTargetLowering::createMemcpyOutsideCallSeq( 5919 SDValue Arg, SDValue PtrOff, SDValue CallSeqStart, ISD::ArgFlagsTy Flags, 5920 SelectionDAG &DAG, const SDLoc &dl) const { 5921 SDValue MemcpyCall = CreateCopyOfByValArgument(Arg, PtrOff, 5922 CallSeqStart.getNode()->getOperand(0), 5923 Flags, DAG, dl); 5924 // The MEMCPY must go outside the CALLSEQ_START..END. 5925 int64_t FrameSize = CallSeqStart.getConstantOperandVal(1); 5926 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall, FrameSize, 0, 5927 SDLoc(MemcpyCall)); 5928 DAG.ReplaceAllUsesWith(CallSeqStart.getNode(), 5929 NewCallSeqStart.getNode()); 5930 return NewCallSeqStart; 5931 } 5932 5933 SDValue PPCTargetLowering::LowerCall_64SVR4( 5934 SDValue Chain, SDValue Callee, CallFlags CFlags, 5935 const SmallVectorImpl<ISD::OutputArg> &Outs, 5936 const SmallVectorImpl<SDValue> &OutVals, 5937 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 5938 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, 5939 const CallBase *CB) const { 5940 bool isELFv2ABI = Subtarget.isELFv2ABI(); 5941 bool isLittleEndian = Subtarget.isLittleEndian(); 5942 unsigned NumOps = Outs.size(); 5943 bool IsSibCall = false; 5944 bool IsFastCall = CFlags.CallConv == CallingConv::Fast; 5945 5946 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 5947 unsigned PtrByteSize = 8; 5948 5949 MachineFunction &MF = DAG.getMachineFunction(); 5950 5951 if (CFlags.IsTailCall && !getTargetMachine().Options.GuaranteedTailCallOpt) 5952 IsSibCall = true; 5953 5954 // Mark this function as potentially containing a function that contains a 5955 // tail call. As a consequence the frame pointer will be used for dynamicalloc 5956 // and restoring the callers stack pointer in this functions epilog. This is 5957 // done because by tail calling the called function might overwrite the value 5958 // in this function's (MF) stack pointer stack slot 0(SP). 5959 if (getTargetMachine().Options.GuaranteedTailCallOpt && IsFastCall) 5960 MF.getInfo<PPCFunctionInfo>()->setHasFastCall(); 5961 5962 assert(!(IsFastCall && CFlags.IsVarArg) && 5963 "fastcc not supported on varargs functions"); 5964 5965 // Count how many bytes are to be pushed on the stack, including the linkage 5966 // area, and parameter passing area. On ELFv1, the linkage area is 48 bytes 5967 // reserved space for [SP][CR][LR][2 x unused][TOC]; on ELFv2, the linkage 5968 // area is 32 bytes reserved space for [SP][CR][LR][TOC]. 5969 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 5970 unsigned NumBytes = LinkageSize; 5971 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; 5972 5973 static const MCPhysReg GPR[] = { 5974 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 5975 PPC::X7, PPC::X8, PPC::X9, PPC::X10, 5976 }; 5977 static const MCPhysReg VR[] = { 5978 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, 5979 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 5980 }; 5981 5982 const unsigned NumGPRs = array_lengthof(GPR); 5983 const unsigned NumFPRs = useSoftFloat() ? 0 : 13; 5984 const unsigned NumVRs = array_lengthof(VR); 5985 5986 // On ELFv2, we can avoid allocating the parameter area if all the arguments 5987 // can be passed to the callee in registers. 5988 // For the fast calling convention, there is another check below. 5989 // Note: We should keep consistent with LowerFormalArguments_64SVR4() 5990 bool HasParameterArea = !isELFv2ABI || CFlags.IsVarArg || IsFastCall; 5991 if (!HasParameterArea) { 5992 unsigned ParamAreaSize = NumGPRs * PtrByteSize; 5993 unsigned AvailableFPRs = NumFPRs; 5994 unsigned AvailableVRs = NumVRs; 5995 unsigned NumBytesTmp = NumBytes; 5996 for (unsigned i = 0; i != NumOps; ++i) { 5997 if (Outs[i].Flags.isNest()) continue; 5998 if (CalculateStackSlotUsed(Outs[i].VT, Outs[i].ArgVT, Outs[i].Flags, 5999 PtrByteSize, LinkageSize, ParamAreaSize, 6000 NumBytesTmp, AvailableFPRs, AvailableVRs)) 6001 HasParameterArea = true; 6002 } 6003 } 6004 6005 // When using the fast calling convention, we don't provide backing for 6006 // arguments that will be in registers. 6007 unsigned NumGPRsUsed = 0, NumFPRsUsed = 0, NumVRsUsed = 0; 6008 6009 // Avoid allocating parameter area for fastcc functions if all the arguments 6010 // can be passed in the registers. 6011 if (IsFastCall) 6012 HasParameterArea = false; 6013 6014 // Add up all the space actually used. 6015 for (unsigned i = 0; i != NumOps; ++i) { 6016 ISD::ArgFlagsTy Flags = Outs[i].Flags; 6017 EVT ArgVT = Outs[i].VT; 6018 EVT OrigVT = Outs[i].ArgVT; 6019 6020 if (Flags.isNest()) 6021 continue; 6022 6023 if (IsFastCall) { 6024 if (Flags.isByVal()) { 6025 NumGPRsUsed += (Flags.getByValSize()+7)/8; 6026 if (NumGPRsUsed > NumGPRs) 6027 HasParameterArea = true; 6028 } else { 6029 switch (ArgVT.getSimpleVT().SimpleTy) { 6030 default: llvm_unreachable("Unexpected ValueType for argument!"); 6031 case MVT::i1: 6032 case MVT::i32: 6033 case MVT::i64: 6034 if (++NumGPRsUsed <= NumGPRs) 6035 continue; 6036 break; 6037 case MVT::v4i32: 6038 case MVT::v8i16: 6039 case MVT::v16i8: 6040 case MVT::v2f64: 6041 case MVT::v2i64: 6042 case MVT::v1i128: 6043 case MVT::f128: 6044 if (++NumVRsUsed <= NumVRs) 6045 continue; 6046 break; 6047 case MVT::v4f32: 6048 if (++NumVRsUsed <= NumVRs) 6049 continue; 6050 break; 6051 case MVT::f32: 6052 case MVT::f64: 6053 if (++NumFPRsUsed <= NumFPRs) 6054 continue; 6055 break; 6056 } 6057 HasParameterArea = true; 6058 } 6059 } 6060 6061 /* Respect alignment of argument on the stack. */ 6062 auto Alignement = 6063 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize); 6064 NumBytes = alignTo(NumBytes, Alignement); 6065 6066 NumBytes += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize); 6067 if (Flags.isInConsecutiveRegsLast()) 6068 NumBytes = ((NumBytes + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 6069 } 6070 6071 unsigned NumBytesActuallyUsed = NumBytes; 6072 6073 // In the old ELFv1 ABI, 6074 // the prolog code of the callee may store up to 8 GPR argument registers to 6075 // the stack, allowing va_start to index over them in memory if its varargs. 6076 // Because we cannot tell if this is needed on the caller side, we have to 6077 // conservatively assume that it is needed. As such, make sure we have at 6078 // least enough stack space for the caller to store the 8 GPRs. 6079 // In the ELFv2 ABI, we allocate the parameter area iff a callee 6080 // really requires memory operands, e.g. a vararg function. 6081 if (HasParameterArea) 6082 NumBytes = std::max(NumBytes, LinkageSize + 8 * PtrByteSize); 6083 else 6084 NumBytes = LinkageSize; 6085 6086 // Tail call needs the stack to be aligned. 6087 if (getTargetMachine().Options.GuaranteedTailCallOpt && IsFastCall) 6088 NumBytes = EnsureStackAlignment(Subtarget.getFrameLowering(), NumBytes); 6089 6090 int SPDiff = 0; 6091 6092 // Calculate by how many bytes the stack has to be adjusted in case of tail 6093 // call optimization. 6094 if (!IsSibCall) 6095 SPDiff = CalculateTailCallSPDiff(DAG, CFlags.IsTailCall, NumBytes); 6096 6097 // To protect arguments on the stack from being clobbered in a tail call, 6098 // force all the loads to happen before doing any other lowering. 6099 if (CFlags.IsTailCall) 6100 Chain = DAG.getStackArgumentTokenFactor(Chain); 6101 6102 // Adjust the stack pointer for the new arguments... 6103 // These operations are automatically eliminated by the prolog/epilog pass 6104 if (!IsSibCall) 6105 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); 6106 SDValue CallSeqStart = Chain; 6107 6108 // Load the return address and frame pointer so it can be move somewhere else 6109 // later. 6110 SDValue LROp, FPOp; 6111 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, dl); 6112 6113 // Set up a copy of the stack pointer for use loading and storing any 6114 // arguments that may not fit in the registers available for argument 6115 // passing. 6116 SDValue StackPtr = DAG.getRegister(PPC::X1, MVT::i64); 6117 6118 // Figure out which arguments are going to go in registers, and which in 6119 // memory. Also, if this is a vararg function, floating point operations 6120 // must be stored to our stack, and loaded into integer regs as well, if 6121 // any integer regs are available for argument passing. 6122 unsigned ArgOffset = LinkageSize; 6123 6124 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 6125 SmallVector<TailCallArgumentInfo, 8> TailCallArguments; 6126 6127 SmallVector<SDValue, 8> MemOpChains; 6128 for (unsigned i = 0; i != NumOps; ++i) { 6129 SDValue Arg = OutVals[i]; 6130 ISD::ArgFlagsTy Flags = Outs[i].Flags; 6131 EVT ArgVT = Outs[i].VT; 6132 EVT OrigVT = Outs[i].ArgVT; 6133 6134 // PtrOff will be used to store the current argument to the stack if a 6135 // register cannot be found for it. 6136 SDValue PtrOff; 6137 6138 // We re-align the argument offset for each argument, except when using the 6139 // fast calling convention, when we need to make sure we do that only when 6140 // we'll actually use a stack slot. 6141 auto ComputePtrOff = [&]() { 6142 /* Respect alignment of argument on the stack. */ 6143 auto Alignment = 6144 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize); 6145 ArgOffset = alignTo(ArgOffset, Alignment); 6146 6147 PtrOff = DAG.getConstant(ArgOffset, dl, StackPtr.getValueType()); 6148 6149 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); 6150 }; 6151 6152 if (!IsFastCall) { 6153 ComputePtrOff(); 6154 6155 /* Compute GPR index associated with argument offset. */ 6156 GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize; 6157 GPR_idx = std::min(GPR_idx, NumGPRs); 6158 } 6159 6160 // Promote integers to 64-bit values. 6161 if (Arg.getValueType() == MVT::i32 || Arg.getValueType() == MVT::i1) { 6162 // FIXME: Should this use ANY_EXTEND if neither sext nor zext? 6163 unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 6164 Arg = DAG.getNode(ExtOp, dl, MVT::i64, Arg); 6165 } 6166 6167 // FIXME memcpy is used way more than necessary. Correctness first. 6168 // Note: "by value" is code for passing a structure by value, not 6169 // basic types. 6170 if (Flags.isByVal()) { 6171 // Note: Size includes alignment padding, so 6172 // struct x { short a; char b; } 6173 // will have Size = 4. With #pragma pack(1), it will have Size = 3. 6174 // These are the proper values we need for right-justifying the 6175 // aggregate in a parameter register. 6176 unsigned Size = Flags.getByValSize(); 6177 6178 // An empty aggregate parameter takes up no storage and no 6179 // registers. 6180 if (Size == 0) 6181 continue; 6182 6183 if (IsFastCall) 6184 ComputePtrOff(); 6185 6186 // All aggregates smaller than 8 bytes must be passed right-justified. 6187 if (Size==1 || Size==2 || Size==4) { 6188 EVT VT = (Size==1) ? MVT::i8 : ((Size==2) ? MVT::i16 : MVT::i32); 6189 if (GPR_idx != NumGPRs) { 6190 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg, 6191 MachinePointerInfo(), VT); 6192 MemOpChains.push_back(Load.getValue(1)); 6193 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 6194 6195 ArgOffset += PtrByteSize; 6196 continue; 6197 } 6198 } 6199 6200 if (GPR_idx == NumGPRs && Size < 8) { 6201 SDValue AddPtr = PtrOff; 6202 if (!isLittleEndian) { 6203 SDValue Const = DAG.getConstant(PtrByteSize - Size, dl, 6204 PtrOff.getValueType()); 6205 AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const); 6206 } 6207 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr, 6208 CallSeqStart, 6209 Flags, DAG, dl); 6210 ArgOffset += PtrByteSize; 6211 continue; 6212 } 6213 // Copy the object to parameter save area if it can not be entirely passed 6214 // by registers. 6215 // FIXME: we only need to copy the parts which need to be passed in 6216 // parameter save area. For the parts passed by registers, we don't need 6217 // to copy them to the stack although we need to allocate space for them 6218 // in parameter save area. 6219 if ((NumGPRs - GPR_idx) * PtrByteSize < Size) 6220 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff, 6221 CallSeqStart, 6222 Flags, DAG, dl); 6223 6224 // When a register is available, pass a small aggregate right-justified. 6225 if (Size < 8 && GPR_idx != NumGPRs) { 6226 // The easiest way to get this right-justified in a register 6227 // is to copy the structure into the rightmost portion of a 6228 // local variable slot, then load the whole slot into the 6229 // register. 6230 // FIXME: The memcpy seems to produce pretty awful code for 6231 // small aggregates, particularly for packed ones. 6232 // FIXME: It would be preferable to use the slot in the 6233 // parameter save area instead of a new local variable. 6234 SDValue AddPtr = PtrOff; 6235 if (!isLittleEndian) { 6236 SDValue Const = DAG.getConstant(8 - Size, dl, PtrOff.getValueType()); 6237 AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const); 6238 } 6239 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr, 6240 CallSeqStart, 6241 Flags, DAG, dl); 6242 6243 // Load the slot into the register. 6244 SDValue Load = 6245 DAG.getLoad(PtrVT, dl, Chain, PtrOff, MachinePointerInfo()); 6246 MemOpChains.push_back(Load.getValue(1)); 6247 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 6248 6249 // Done with this argument. 6250 ArgOffset += PtrByteSize; 6251 continue; 6252 } 6253 6254 // For aggregates larger than PtrByteSize, copy the pieces of the 6255 // object that fit into registers from the parameter save area. 6256 for (unsigned j=0; j<Size; j+=PtrByteSize) { 6257 SDValue Const = DAG.getConstant(j, dl, PtrOff.getValueType()); 6258 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const); 6259 if (GPR_idx != NumGPRs) { 6260 SDValue Load = 6261 DAG.getLoad(PtrVT, dl, Chain, AddArg, MachinePointerInfo()); 6262 MemOpChains.push_back(Load.getValue(1)); 6263 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 6264 ArgOffset += PtrByteSize; 6265 } else { 6266 ArgOffset += ((Size - j + PtrByteSize-1)/PtrByteSize)*PtrByteSize; 6267 break; 6268 } 6269 } 6270 continue; 6271 } 6272 6273 switch (Arg.getSimpleValueType().SimpleTy) { 6274 default: llvm_unreachable("Unexpected ValueType for argument!"); 6275 case MVT::i1: 6276 case MVT::i32: 6277 case MVT::i64: 6278 if (Flags.isNest()) { 6279 // The 'nest' parameter, if any, is passed in R11. 6280 RegsToPass.push_back(std::make_pair(PPC::X11, Arg)); 6281 break; 6282 } 6283 6284 // These can be scalar arguments or elements of an integer array type 6285 // passed directly. Clang may use those instead of "byval" aggregate 6286 // types to avoid forcing arguments to memory unnecessarily. 6287 if (GPR_idx != NumGPRs) { 6288 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Arg)); 6289 } else { 6290 if (IsFastCall) 6291 ComputePtrOff(); 6292 6293 assert(HasParameterArea && 6294 "Parameter area must exist to pass an argument in memory."); 6295 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 6296 true, CFlags.IsTailCall, false, MemOpChains, 6297 TailCallArguments, dl); 6298 if (IsFastCall) 6299 ArgOffset += PtrByteSize; 6300 } 6301 if (!IsFastCall) 6302 ArgOffset += PtrByteSize; 6303 break; 6304 case MVT::f32: 6305 case MVT::f64: { 6306 // These can be scalar arguments or elements of a float array type 6307 // passed directly. The latter are used to implement ELFv2 homogenous 6308 // float aggregates. 6309 6310 // Named arguments go into FPRs first, and once they overflow, the 6311 // remaining arguments go into GPRs and then the parameter save area. 6312 // Unnamed arguments for vararg functions always go to GPRs and 6313 // then the parameter save area. For now, put all arguments to vararg 6314 // routines always in both locations (FPR *and* GPR or stack slot). 6315 bool NeedGPROrStack = CFlags.IsVarArg || FPR_idx == NumFPRs; 6316 bool NeededLoad = false; 6317 6318 // First load the argument into the next available FPR. 6319 if (FPR_idx != NumFPRs) 6320 RegsToPass.push_back(std::make_pair(FPR[FPR_idx++], Arg)); 6321 6322 // Next, load the argument into GPR or stack slot if needed. 6323 if (!NeedGPROrStack) 6324 ; 6325 else if (GPR_idx != NumGPRs && !IsFastCall) { 6326 // FIXME: We may want to re-enable this for CallingConv::Fast on the P8 6327 // once we support fp <-> gpr moves. 6328 6329 // In the non-vararg case, this can only ever happen in the 6330 // presence of f32 array types, since otherwise we never run 6331 // out of FPRs before running out of GPRs. 6332 SDValue ArgVal; 6333 6334 // Double values are always passed in a single GPR. 6335 if (Arg.getValueType() != MVT::f32) { 6336 ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i64, Arg); 6337 6338 // Non-array float values are extended and passed in a GPR. 6339 } else if (!Flags.isInConsecutiveRegs()) { 6340 ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg); 6341 ArgVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, ArgVal); 6342 6343 // If we have an array of floats, we collect every odd element 6344 // together with its predecessor into one GPR. 6345 } else if (ArgOffset % PtrByteSize != 0) { 6346 SDValue Lo, Hi; 6347 Lo = DAG.getNode(ISD::BITCAST, dl, MVT::i32, OutVals[i - 1]); 6348 Hi = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg); 6349 if (!isLittleEndian) 6350 std::swap(Lo, Hi); 6351 ArgVal = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi); 6352 6353 // The final element, if even, goes into the first half of a GPR. 6354 } else if (Flags.isInConsecutiveRegsLast()) { 6355 ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg); 6356 ArgVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, ArgVal); 6357 if (!isLittleEndian) 6358 ArgVal = DAG.getNode(ISD::SHL, dl, MVT::i64, ArgVal, 6359 DAG.getConstant(32, dl, MVT::i32)); 6360 6361 // Non-final even elements are skipped; they will be handled 6362 // together the with subsequent argument on the next go-around. 6363 } else 6364 ArgVal = SDValue(); 6365 6366 if (ArgVal.getNode()) 6367 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], ArgVal)); 6368 } else { 6369 if (IsFastCall) 6370 ComputePtrOff(); 6371 6372 // Single-precision floating-point values are mapped to the 6373 // second (rightmost) word of the stack doubleword. 6374 if (Arg.getValueType() == MVT::f32 && 6375 !isLittleEndian && !Flags.isInConsecutiveRegs()) { 6376 SDValue ConstFour = DAG.getConstant(4, dl, PtrOff.getValueType()); 6377 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, ConstFour); 6378 } 6379 6380 assert(HasParameterArea && 6381 "Parameter area must exist to pass an argument in memory."); 6382 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 6383 true, CFlags.IsTailCall, false, MemOpChains, 6384 TailCallArguments, dl); 6385 6386 NeededLoad = true; 6387 } 6388 // When passing an array of floats, the array occupies consecutive 6389 // space in the argument area; only round up to the next doubleword 6390 // at the end of the array. Otherwise, each float takes 8 bytes. 6391 if (!IsFastCall || NeededLoad) { 6392 ArgOffset += (Arg.getValueType() == MVT::f32 && 6393 Flags.isInConsecutiveRegs()) ? 4 : 8; 6394 if (Flags.isInConsecutiveRegsLast()) 6395 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 6396 } 6397 break; 6398 } 6399 case MVT::v4f32: 6400 case MVT::v4i32: 6401 case MVT::v8i16: 6402 case MVT::v16i8: 6403 case MVT::v2f64: 6404 case MVT::v2i64: 6405 case MVT::v1i128: 6406 case MVT::f128: 6407 // These can be scalar arguments or elements of a vector array type 6408 // passed directly. The latter are used to implement ELFv2 homogenous 6409 // vector aggregates. 6410 6411 // For a varargs call, named arguments go into VRs or on the stack as 6412 // usual; unnamed arguments always go to the stack or the corresponding 6413 // GPRs when within range. For now, we always put the value in both 6414 // locations (or even all three). 6415 if (CFlags.IsVarArg) { 6416 assert(HasParameterArea && 6417 "Parameter area must exist if we have a varargs call."); 6418 // We could elide this store in the case where the object fits 6419 // entirely in R registers. Maybe later. 6420 SDValue Store = 6421 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()); 6422 MemOpChains.push_back(Store); 6423 if (VR_idx != NumVRs) { 6424 SDValue Load = 6425 DAG.getLoad(MVT::v4f32, dl, Store, PtrOff, MachinePointerInfo()); 6426 MemOpChains.push_back(Load.getValue(1)); 6427 RegsToPass.push_back(std::make_pair(VR[VR_idx++], Load)); 6428 } 6429 ArgOffset += 16; 6430 for (unsigned i=0; i<16; i+=PtrByteSize) { 6431 if (GPR_idx == NumGPRs) 6432 break; 6433 SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, 6434 DAG.getConstant(i, dl, PtrVT)); 6435 SDValue Load = 6436 DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo()); 6437 MemOpChains.push_back(Load.getValue(1)); 6438 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 6439 } 6440 break; 6441 } 6442 6443 // Non-varargs Altivec params go into VRs or on the stack. 6444 if (VR_idx != NumVRs) { 6445 RegsToPass.push_back(std::make_pair(VR[VR_idx++], Arg)); 6446 } else { 6447 if (IsFastCall) 6448 ComputePtrOff(); 6449 6450 assert(HasParameterArea && 6451 "Parameter area must exist to pass an argument in memory."); 6452 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 6453 true, CFlags.IsTailCall, true, MemOpChains, 6454 TailCallArguments, dl); 6455 if (IsFastCall) 6456 ArgOffset += 16; 6457 } 6458 6459 if (!IsFastCall) 6460 ArgOffset += 16; 6461 break; 6462 } 6463 } 6464 6465 assert((!HasParameterArea || NumBytesActuallyUsed == ArgOffset) && 6466 "mismatch in size of parameter area"); 6467 (void)NumBytesActuallyUsed; 6468 6469 if (!MemOpChains.empty()) 6470 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); 6471 6472 // Check if this is an indirect call (MTCTR/BCTRL). 6473 // See prepareDescriptorIndirectCall and buildCallOperands for more 6474 // information about calls through function pointers in the 64-bit SVR4 ABI. 6475 if (CFlags.IsIndirect) { 6476 // For 64-bit ELFv2 ABI with PCRel, do not save the TOC of the 6477 // caller in the TOC save area. 6478 if (isTOCSaveRestoreRequired(Subtarget)) { 6479 assert(!CFlags.IsTailCall && "Indirect tails calls not supported"); 6480 // Load r2 into a virtual register and store it to the TOC save area. 6481 setUsesTOCBasePtr(DAG); 6482 SDValue Val = DAG.getCopyFromReg(Chain, dl, PPC::X2, MVT::i64); 6483 // TOC save area offset. 6484 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset(); 6485 SDValue PtrOff = DAG.getIntPtrConstant(TOCSaveOffset, dl); 6486 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); 6487 Chain = DAG.getStore(Val.getValue(1), dl, Val, AddPtr, 6488 MachinePointerInfo::getStack( 6489 DAG.getMachineFunction(), TOCSaveOffset)); 6490 } 6491 // In the ELFv2 ABI, R12 must contain the address of an indirect callee. 6492 // This does not mean the MTCTR instruction must use R12; it's easier 6493 // to model this as an extra parameter, so do that. 6494 if (isELFv2ABI && !CFlags.IsPatchPoint) 6495 RegsToPass.push_back(std::make_pair((unsigned)PPC::X12, Callee)); 6496 } 6497 6498 // Build a sequence of copy-to-reg nodes chained together with token chain 6499 // and flag operands which copy the outgoing args into the appropriate regs. 6500 SDValue InFlag; 6501 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 6502 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 6503 RegsToPass[i].second, InFlag); 6504 InFlag = Chain.getValue(1); 6505 } 6506 6507 if (CFlags.IsTailCall && !IsSibCall) 6508 PrepareTailCall(DAG, InFlag, Chain, dl, SPDiff, NumBytes, LROp, FPOp, 6509 TailCallArguments); 6510 6511 return FinishCall(CFlags, dl, DAG, RegsToPass, InFlag, Chain, CallSeqStart, 6512 Callee, SPDiff, NumBytes, Ins, InVals, CB); 6513 } 6514 6515 // Returns true when the shadow of a general purpose argument register 6516 // in the parameter save area is aligned to at least 'RequiredAlign'. 6517 static bool isGPRShadowAligned(MCPhysReg Reg, Align RequiredAlign) { 6518 assert(RequiredAlign.value() <= 16 && 6519 "Required alignment greater than stack alignment."); 6520 switch (Reg) { 6521 default: 6522 report_fatal_error("called on invalid register."); 6523 case PPC::R5: 6524 case PPC::R9: 6525 case PPC::X3: 6526 case PPC::X5: 6527 case PPC::X7: 6528 case PPC::X9: 6529 // These registers are 16 byte aligned which is the most strict aligment 6530 // we can support. 6531 return true; 6532 case PPC::R3: 6533 case PPC::R7: 6534 case PPC::X4: 6535 case PPC::X6: 6536 case PPC::X8: 6537 case PPC::X10: 6538 // The shadow of these registers in the PSA is 8 byte aligned. 6539 return RequiredAlign <= 8; 6540 case PPC::R4: 6541 case PPC::R6: 6542 case PPC::R8: 6543 case PPC::R10: 6544 return RequiredAlign <= 4; 6545 } 6546 } 6547 6548 static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT, 6549 CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, 6550 CCState &S) { 6551 AIXCCState &State = static_cast<AIXCCState &>(S); 6552 const PPCSubtarget &Subtarget = static_cast<const PPCSubtarget &>( 6553 State.getMachineFunction().getSubtarget()); 6554 const bool IsPPC64 = Subtarget.isPPC64(); 6555 const Align PtrAlign = IsPPC64 ? Align(8) : Align(4); 6556 const MVT RegVT = IsPPC64 ? MVT::i64 : MVT::i32; 6557 6558 if (ValVT == MVT::f128) 6559 report_fatal_error("f128 is unimplemented on AIX."); 6560 6561 if (ArgFlags.isNest()) 6562 report_fatal_error("Nest arguments are unimplemented."); 6563 6564 static const MCPhysReg GPR_32[] = {// 32-bit registers. 6565 PPC::R3, PPC::R4, PPC::R5, PPC::R6, 6566 PPC::R7, PPC::R8, PPC::R9, PPC::R10}; 6567 static const MCPhysReg GPR_64[] = {// 64-bit registers. 6568 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 6569 PPC::X7, PPC::X8, PPC::X9, PPC::X10}; 6570 6571 static const MCPhysReg VR[] = {// Vector registers. 6572 PPC::V2, PPC::V3, PPC::V4, PPC::V5, 6573 PPC::V6, PPC::V7, PPC::V8, PPC::V9, 6574 PPC::V10, PPC::V11, PPC::V12, PPC::V13}; 6575 6576 if (ArgFlags.isByVal()) { 6577 if (ArgFlags.getNonZeroByValAlign() > PtrAlign) 6578 report_fatal_error("Pass-by-value arguments with alignment greater than " 6579 "register width are not supported."); 6580 6581 const unsigned ByValSize = ArgFlags.getByValSize(); 6582 6583 // An empty aggregate parameter takes up no storage and no registers, 6584 // but needs a MemLoc for a stack slot for the formal arguments side. 6585 if (ByValSize == 0) { 6586 State.addLoc(CCValAssign::getMem(ValNo, MVT::INVALID_SIMPLE_VALUE_TYPE, 6587 State.getNextStackOffset(), RegVT, 6588 LocInfo)); 6589 return false; 6590 } 6591 6592 const unsigned StackSize = alignTo(ByValSize, PtrAlign); 6593 unsigned Offset = State.AllocateStack(StackSize, PtrAlign); 6594 for (const unsigned E = Offset + StackSize; Offset < E; 6595 Offset += PtrAlign.value()) { 6596 if (unsigned Reg = State.AllocateReg(IsPPC64 ? GPR_64 : GPR_32)) 6597 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, RegVT, LocInfo)); 6598 else { 6599 State.addLoc(CCValAssign::getMem(ValNo, MVT::INVALID_SIMPLE_VALUE_TYPE, 6600 Offset, MVT::INVALID_SIMPLE_VALUE_TYPE, 6601 LocInfo)); 6602 break; 6603 } 6604 } 6605 return false; 6606 } 6607 6608 // Arguments always reserve parameter save area. 6609 switch (ValVT.SimpleTy) { 6610 default: 6611 report_fatal_error("Unhandled value type for argument."); 6612 case MVT::i64: 6613 // i64 arguments should have been split to i32 for PPC32. 6614 assert(IsPPC64 && "PPC32 should have split i64 values."); 6615 LLVM_FALLTHROUGH; 6616 case MVT::i1: 6617 case MVT::i32: { 6618 const unsigned Offset = State.AllocateStack(PtrAlign.value(), PtrAlign); 6619 // AIX integer arguments are always passed in register width. 6620 if (ValVT.getFixedSizeInBits() < RegVT.getFixedSizeInBits()) 6621 LocInfo = ArgFlags.isSExt() ? CCValAssign::LocInfo::SExt 6622 : CCValAssign::LocInfo::ZExt; 6623 if (unsigned Reg = State.AllocateReg(IsPPC64 ? GPR_64 : GPR_32)) 6624 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, RegVT, LocInfo)); 6625 else 6626 State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, RegVT, LocInfo)); 6627 6628 return false; 6629 } 6630 case MVT::f32: 6631 case MVT::f64: { 6632 // Parameter save area (PSA) is reserved even if the float passes in fpr. 6633 const unsigned StoreSize = LocVT.getStoreSize(); 6634 // Floats are always 4-byte aligned in the PSA on AIX. 6635 // This includes f64 in 64-bit mode for ABI compatibility. 6636 const unsigned Offset = 6637 State.AllocateStack(IsPPC64 ? 8 : StoreSize, Align(4)); 6638 unsigned FReg = State.AllocateReg(FPR); 6639 if (FReg) 6640 State.addLoc(CCValAssign::getReg(ValNo, ValVT, FReg, LocVT, LocInfo)); 6641 6642 // Reserve and initialize GPRs or initialize the PSA as required. 6643 for (unsigned I = 0; I < StoreSize; I += PtrAlign.value()) { 6644 if (unsigned Reg = State.AllocateReg(IsPPC64 ? GPR_64 : GPR_32)) { 6645 assert(FReg && "An FPR should be available when a GPR is reserved."); 6646 if (State.isVarArg()) { 6647 // Successfully reserved GPRs are only initialized for vararg calls. 6648 // Custom handling is required for: 6649 // f64 in PPC32 needs to be split into 2 GPRs. 6650 // f32 in PPC64 needs to occupy only lower 32 bits of 64-bit GPR. 6651 State.addLoc( 6652 CCValAssign::getCustomReg(ValNo, ValVT, Reg, RegVT, LocInfo)); 6653 } 6654 } else { 6655 // If there are insufficient GPRs, the PSA needs to be initialized. 6656 // Initialization occurs even if an FPR was initialized for 6657 // compatibility with the AIX XL compiler. The full memory for the 6658 // argument will be initialized even if a prior word is saved in GPR. 6659 // A custom memLoc is used when the argument also passes in FPR so 6660 // that the callee handling can skip over it easily. 6661 State.addLoc( 6662 FReg ? CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT, 6663 LocInfo) 6664 : CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo)); 6665 break; 6666 } 6667 } 6668 6669 return false; 6670 } 6671 case MVT::v4f32: 6672 case MVT::v4i32: 6673 case MVT::v8i16: 6674 case MVT::v16i8: 6675 case MVT::v2i64: 6676 case MVT::v2f64: 6677 case MVT::v1i128: { 6678 const unsigned VecSize = 16; 6679 const Align VecAlign(VecSize); 6680 6681 if (!State.isVarArg()) { 6682 // If there are vector registers remaining we don't consume any stack 6683 // space. 6684 if (unsigned VReg = State.AllocateReg(VR)) { 6685 State.addLoc(CCValAssign::getReg(ValNo, ValVT, VReg, LocVT, LocInfo)); 6686 return false; 6687 } 6688 // Vectors passed on the stack do not shadow GPRs or FPRs even though they 6689 // might be allocated in the portion of the PSA that is shadowed by the 6690 // GPRs. 6691 const unsigned Offset = State.AllocateStack(VecSize, VecAlign); 6692 State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo)); 6693 return false; 6694 } 6695 6696 const unsigned PtrSize = IsPPC64 ? 8 : 4; 6697 ArrayRef<MCPhysReg> GPRs = IsPPC64 ? GPR_64 : GPR_32; 6698 6699 unsigned NextRegIndex = State.getFirstUnallocated(GPRs); 6700 // Burn any underaligned registers and their shadowed stack space until 6701 // we reach the required alignment. 6702 while (NextRegIndex != GPRs.size() && 6703 !isGPRShadowAligned(GPRs[NextRegIndex], VecAlign)) { 6704 // Shadow allocate register and its stack shadow. 6705 unsigned Reg = State.AllocateReg(GPRs); 6706 State.AllocateStack(PtrSize, PtrAlign); 6707 assert(Reg && "Allocating register unexpectedly failed."); 6708 (void)Reg; 6709 NextRegIndex = State.getFirstUnallocated(GPRs); 6710 } 6711 6712 // Vectors that are passed as fixed arguments are handled differently. 6713 // They are passed in VRs if any are available (unlike arguments passed 6714 // through ellipses) and shadow GPRs (unlike arguments to non-vaarg 6715 // functions) 6716 if (State.isFixed(ValNo)) { 6717 if (unsigned VReg = State.AllocateReg(VR)) { 6718 State.addLoc(CCValAssign::getReg(ValNo, ValVT, VReg, LocVT, LocInfo)); 6719 // Shadow allocate GPRs and stack space even though we pass in a VR. 6720 for (unsigned I = 0; I != VecSize; I += PtrSize) 6721 State.AllocateReg(GPRs); 6722 State.AllocateStack(VecSize, VecAlign); 6723 return false; 6724 } 6725 // No vector registers remain so pass on the stack. 6726 const unsigned Offset = State.AllocateStack(VecSize, VecAlign); 6727 State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo)); 6728 return false; 6729 } 6730 6731 // If all GPRS are consumed then we pass the argument fully on the stack. 6732 if (NextRegIndex == GPRs.size()) { 6733 const unsigned Offset = State.AllocateStack(VecSize, VecAlign); 6734 State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo)); 6735 return false; 6736 } 6737 6738 // Corner case for 32-bit codegen. We have 2 registers to pass the first 6739 // half of the argument, and then need to pass the remaining half on the 6740 // stack. 6741 if (GPRs[NextRegIndex] == PPC::R9) { 6742 const unsigned Offset = State.AllocateStack(VecSize, VecAlign); 6743 State.addLoc( 6744 CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT, LocInfo)); 6745 6746 const unsigned FirstReg = State.AllocateReg(PPC::R9); 6747 const unsigned SecondReg = State.AllocateReg(PPC::R10); 6748 assert(FirstReg && SecondReg && 6749 "Allocating R9 or R10 unexpectedly failed."); 6750 State.addLoc( 6751 CCValAssign::getCustomReg(ValNo, ValVT, FirstReg, RegVT, LocInfo)); 6752 State.addLoc( 6753 CCValAssign::getCustomReg(ValNo, ValVT, SecondReg, RegVT, LocInfo)); 6754 return false; 6755 } 6756 6757 // We have enough GPRs to fully pass the vector argument, and we have 6758 // already consumed any underaligned registers. Start with the custom 6759 // MemLoc and then the custom RegLocs. 6760 const unsigned Offset = State.AllocateStack(VecSize, VecAlign); 6761 State.addLoc( 6762 CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT, LocInfo)); 6763 for (unsigned I = 0; I != VecSize; I += PtrSize) { 6764 const unsigned Reg = State.AllocateReg(GPRs); 6765 assert(Reg && "Failed to allocated register for vararg vector argument"); 6766 State.addLoc( 6767 CCValAssign::getCustomReg(ValNo, ValVT, Reg, RegVT, LocInfo)); 6768 } 6769 return false; 6770 } 6771 } 6772 return true; 6773 } 6774 6775 // So far, this function is only used by LowerFormalArguments_AIX() 6776 static const TargetRegisterClass *getRegClassForSVT(MVT::SimpleValueType SVT, 6777 bool IsPPC64, 6778 bool HasP8Vector, 6779 bool HasVSX) { 6780 assert((IsPPC64 || SVT != MVT::i64) && 6781 "i64 should have been split for 32-bit codegen."); 6782 6783 switch (SVT) { 6784 default: 6785 report_fatal_error("Unexpected value type for formal argument"); 6786 case MVT::i1: 6787 case MVT::i32: 6788 case MVT::i64: 6789 return IsPPC64 ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; 6790 case MVT::f32: 6791 return HasP8Vector ? &PPC::VSSRCRegClass : &PPC::F4RCRegClass; 6792 case MVT::f64: 6793 return HasVSX ? &PPC::VSFRCRegClass : &PPC::F8RCRegClass; 6794 case MVT::v4f32: 6795 case MVT::v4i32: 6796 case MVT::v8i16: 6797 case MVT::v16i8: 6798 case MVT::v2i64: 6799 case MVT::v2f64: 6800 case MVT::v1i128: 6801 return &PPC::VRRCRegClass; 6802 } 6803 } 6804 6805 static SDValue truncateScalarIntegerArg(ISD::ArgFlagsTy Flags, EVT ValVT, 6806 SelectionDAG &DAG, SDValue ArgValue, 6807 MVT LocVT, const SDLoc &dl) { 6808 assert(ValVT.isScalarInteger() && LocVT.isScalarInteger()); 6809 assert(ValVT.getFixedSizeInBits() < LocVT.getFixedSizeInBits()); 6810 6811 if (Flags.isSExt()) 6812 ArgValue = DAG.getNode(ISD::AssertSext, dl, LocVT, ArgValue, 6813 DAG.getValueType(ValVT)); 6814 else if (Flags.isZExt()) 6815 ArgValue = DAG.getNode(ISD::AssertZext, dl, LocVT, ArgValue, 6816 DAG.getValueType(ValVT)); 6817 6818 return DAG.getNode(ISD::TRUNCATE, dl, ValVT, ArgValue); 6819 } 6820 6821 static unsigned mapArgRegToOffsetAIX(unsigned Reg, const PPCFrameLowering *FL) { 6822 const unsigned LASize = FL->getLinkageSize(); 6823 6824 if (PPC::GPRCRegClass.contains(Reg)) { 6825 assert(Reg >= PPC::R3 && Reg <= PPC::R10 && 6826 "Reg must be a valid argument register!"); 6827 return LASize + 4 * (Reg - PPC::R3); 6828 } 6829 6830 if (PPC::G8RCRegClass.contains(Reg)) { 6831 assert(Reg >= PPC::X3 && Reg <= PPC::X10 && 6832 "Reg must be a valid argument register!"); 6833 return LASize + 8 * (Reg - PPC::X3); 6834 } 6835 6836 llvm_unreachable("Only general purpose registers expected."); 6837 } 6838 6839 // AIX ABI Stack Frame Layout: 6840 // 6841 // Low Memory +--------------------------------------------+ 6842 // SP +---> | Back chain | ---+ 6843 // | +--------------------------------------------+ | 6844 // | | Saved Condition Register | | 6845 // | +--------------------------------------------+ | 6846 // | | Saved Linkage Register | | 6847 // | +--------------------------------------------+ | Linkage Area 6848 // | | Reserved for compilers | | 6849 // | +--------------------------------------------+ | 6850 // | | Reserved for binders | | 6851 // | +--------------------------------------------+ | 6852 // | | Saved TOC pointer | ---+ 6853 // | +--------------------------------------------+ 6854 // | | Parameter save area | 6855 // | +--------------------------------------------+ 6856 // | | Alloca space | 6857 // | +--------------------------------------------+ 6858 // | | Local variable space | 6859 // | +--------------------------------------------+ 6860 // | | Float/int conversion temporary | 6861 // | +--------------------------------------------+ 6862 // | | Save area for AltiVec registers | 6863 // | +--------------------------------------------+ 6864 // | | AltiVec alignment padding | 6865 // | +--------------------------------------------+ 6866 // | | Save area for VRSAVE register | 6867 // | +--------------------------------------------+ 6868 // | | Save area for General Purpose registers | 6869 // | +--------------------------------------------+ 6870 // | | Save area for Floating Point registers | 6871 // | +--------------------------------------------+ 6872 // +---- | Back chain | 6873 // High Memory +--------------------------------------------+ 6874 // 6875 // Specifications: 6876 // AIX 7.2 Assembler Language Reference 6877 // Subroutine linkage convention 6878 6879 SDValue PPCTargetLowering::LowerFormalArguments_AIX( 6880 SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 6881 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 6882 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 6883 6884 assert((CallConv == CallingConv::C || CallConv == CallingConv::Cold || 6885 CallConv == CallingConv::Fast) && 6886 "Unexpected calling convention!"); 6887 6888 if (getTargetMachine().Options.GuaranteedTailCallOpt) 6889 report_fatal_error("Tail call support is unimplemented on AIX."); 6890 6891 if (useSoftFloat()) 6892 report_fatal_error("Soft float support is unimplemented on AIX."); 6893 6894 const PPCSubtarget &Subtarget = 6895 static_cast<const PPCSubtarget &>(DAG.getSubtarget()); 6896 6897 const bool IsPPC64 = Subtarget.isPPC64(); 6898 const unsigned PtrByteSize = IsPPC64 ? 8 : 4; 6899 6900 // Assign locations to all of the incoming arguments. 6901 SmallVector<CCValAssign, 16> ArgLocs; 6902 MachineFunction &MF = DAG.getMachineFunction(); 6903 MachineFrameInfo &MFI = MF.getFrameInfo(); 6904 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 6905 AIXCCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext()); 6906 6907 const EVT PtrVT = getPointerTy(MF.getDataLayout()); 6908 // Reserve space for the linkage area on the stack. 6909 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 6910 CCInfo.AllocateStack(LinkageSize, Align(PtrByteSize)); 6911 CCInfo.AnalyzeFormalArguments(Ins, CC_AIX); 6912 6913 SmallVector<SDValue, 8> MemOps; 6914 6915 for (size_t I = 0, End = ArgLocs.size(); I != End; /* No increment here */) { 6916 CCValAssign &VA = ArgLocs[I++]; 6917 MVT LocVT = VA.getLocVT(); 6918 MVT ValVT = VA.getValVT(); 6919 ISD::ArgFlagsTy Flags = Ins[VA.getValNo()].Flags; 6920 // For compatibility with the AIX XL compiler, the float args in the 6921 // parameter save area are initialized even if the argument is available 6922 // in register. The caller is required to initialize both the register 6923 // and memory, however, the callee can choose to expect it in either. 6924 // The memloc is dismissed here because the argument is retrieved from 6925 // the register. 6926 if (VA.isMemLoc() && VA.needsCustom() && ValVT.isFloatingPoint()) 6927 continue; 6928 6929 auto HandleMemLoc = [&]() { 6930 const unsigned LocSize = LocVT.getStoreSize(); 6931 const unsigned ValSize = ValVT.getStoreSize(); 6932 assert((ValSize <= LocSize) && 6933 "Object size is larger than size of MemLoc"); 6934 int CurArgOffset = VA.getLocMemOffset(); 6935 // Objects are right-justified because AIX is big-endian. 6936 if (LocSize > ValSize) 6937 CurArgOffset += LocSize - ValSize; 6938 // Potential tail calls could cause overwriting of argument stack slots. 6939 const bool IsImmutable = 6940 !(getTargetMachine().Options.GuaranteedTailCallOpt && 6941 (CallConv == CallingConv::Fast)); 6942 int FI = MFI.CreateFixedObject(ValSize, CurArgOffset, IsImmutable); 6943 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 6944 SDValue ArgValue = 6945 DAG.getLoad(ValVT, dl, Chain, FIN, MachinePointerInfo()); 6946 InVals.push_back(ArgValue); 6947 }; 6948 6949 // Vector arguments to VaArg functions are passed both on the stack, and 6950 // in any available GPRs. Load the value from the stack and add the GPRs 6951 // as live ins. 6952 if (VA.isMemLoc() && VA.needsCustom()) { 6953 assert(ValVT.isVector() && "Unexpected Custom MemLoc type."); 6954 assert(isVarArg && "Only use custom memloc for vararg."); 6955 // ValNo of the custom MemLoc, so we can compare it to the ValNo of the 6956 // matching custom RegLocs. 6957 const unsigned OriginalValNo = VA.getValNo(); 6958 (void)OriginalValNo; 6959 6960 auto HandleCustomVecRegLoc = [&]() { 6961 assert(I != End && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() && 6962 "Missing custom RegLoc."); 6963 VA = ArgLocs[I++]; 6964 assert(VA.getValVT().isVector() && 6965 "Unexpected Val type for custom RegLoc."); 6966 assert(VA.getValNo() == OriginalValNo && 6967 "ValNo mismatch between custom MemLoc and RegLoc."); 6968 MVT::SimpleValueType SVT = VA.getLocVT().SimpleTy; 6969 MF.addLiveIn(VA.getLocReg(), 6970 getRegClassForSVT(SVT, IsPPC64, Subtarget.hasP8Vector(), 6971 Subtarget.hasVSX())); 6972 }; 6973 6974 HandleMemLoc(); 6975 // In 64-bit there will be exactly 2 custom RegLocs that follow, and in 6976 // in 32-bit there will be 2 custom RegLocs if we are passing in R9 and 6977 // R10. 6978 HandleCustomVecRegLoc(); 6979 HandleCustomVecRegLoc(); 6980 6981 // If we are targeting 32-bit, there might be 2 extra custom RegLocs if 6982 // we passed the vector in R5, R6, R7 and R8. 6983 if (I != End && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom()) { 6984 assert(!IsPPC64 && 6985 "Only 2 custom RegLocs expected for 64-bit codegen."); 6986 HandleCustomVecRegLoc(); 6987 HandleCustomVecRegLoc(); 6988 } 6989 6990 continue; 6991 } 6992 6993 if (VA.isRegLoc()) { 6994 if (VA.getValVT().isScalarInteger()) 6995 FuncInfo->appendParameterType(PPCFunctionInfo::FixedType); 6996 else if (VA.getValVT().isFloatingPoint() && !VA.getValVT().isVector()) { 6997 switch (VA.getValVT().SimpleTy) { 6998 default: 6999 report_fatal_error("Unhandled value type for argument."); 7000 case MVT::f32: 7001 FuncInfo->appendParameterType(PPCFunctionInfo::ShortFloatingPoint); 7002 break; 7003 case MVT::f64: 7004 FuncInfo->appendParameterType(PPCFunctionInfo::LongFloatingPoint); 7005 break; 7006 } 7007 } else if (VA.getValVT().isVector()) { 7008 switch (VA.getValVT().SimpleTy) { 7009 default: 7010 report_fatal_error("Unhandled value type for argument."); 7011 case MVT::v16i8: 7012 FuncInfo->appendParameterType(PPCFunctionInfo::VectorChar); 7013 break; 7014 case MVT::v8i16: 7015 FuncInfo->appendParameterType(PPCFunctionInfo::VectorShort); 7016 break; 7017 case MVT::v4i32: 7018 case MVT::v2i64: 7019 case MVT::v1i128: 7020 FuncInfo->appendParameterType(PPCFunctionInfo::VectorInt); 7021 break; 7022 case MVT::v4f32: 7023 case MVT::v2f64: 7024 FuncInfo->appendParameterType(PPCFunctionInfo::VectorFloat); 7025 break; 7026 } 7027 } 7028 } 7029 7030 if (Flags.isByVal() && VA.isMemLoc()) { 7031 const unsigned Size = 7032 alignTo(Flags.getByValSize() ? Flags.getByValSize() : PtrByteSize, 7033 PtrByteSize); 7034 const int FI = MF.getFrameInfo().CreateFixedObject( 7035 Size, VA.getLocMemOffset(), /* IsImmutable */ false, 7036 /* IsAliased */ true); 7037 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 7038 InVals.push_back(FIN); 7039 7040 continue; 7041 } 7042 7043 if (Flags.isByVal()) { 7044 assert(VA.isRegLoc() && "MemLocs should already be handled."); 7045 7046 const MCPhysReg ArgReg = VA.getLocReg(); 7047 const PPCFrameLowering *FL = Subtarget.getFrameLowering(); 7048 7049 if (Flags.getNonZeroByValAlign() > PtrByteSize) 7050 report_fatal_error("Over aligned byvals not supported yet."); 7051 7052 const unsigned StackSize = alignTo(Flags.getByValSize(), PtrByteSize); 7053 const int FI = MF.getFrameInfo().CreateFixedObject( 7054 StackSize, mapArgRegToOffsetAIX(ArgReg, FL), /* IsImmutable */ false, 7055 /* IsAliased */ true); 7056 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 7057 InVals.push_back(FIN); 7058 7059 // Add live ins for all the RegLocs for the same ByVal. 7060 const TargetRegisterClass *RegClass = 7061 IsPPC64 ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; 7062 7063 auto HandleRegLoc = [&, RegClass, LocVT](const MCPhysReg PhysReg, 7064 unsigned Offset) { 7065 const Register VReg = MF.addLiveIn(PhysReg, RegClass); 7066 // Since the callers side has left justified the aggregate in the 7067 // register, we can simply store the entire register into the stack 7068 // slot. 7069 SDValue CopyFrom = DAG.getCopyFromReg(Chain, dl, VReg, LocVT); 7070 // The store to the fixedstack object is needed becuase accessing a 7071 // field of the ByVal will use a gep and load. Ideally we will optimize 7072 // to extracting the value from the register directly, and elide the 7073 // stores when the arguments address is not taken, but that will need to 7074 // be future work. 7075 SDValue Store = DAG.getStore( 7076 CopyFrom.getValue(1), dl, CopyFrom, 7077 DAG.getObjectPtrOffset(dl, FIN, TypeSize::Fixed(Offset)), 7078 MachinePointerInfo::getFixedStack(MF, FI, Offset)); 7079 7080 MemOps.push_back(Store); 7081 }; 7082 7083 unsigned Offset = 0; 7084 HandleRegLoc(VA.getLocReg(), Offset); 7085 Offset += PtrByteSize; 7086 for (; Offset != StackSize && ArgLocs[I].isRegLoc(); 7087 Offset += PtrByteSize) { 7088 assert(ArgLocs[I].getValNo() == VA.getValNo() && 7089 "RegLocs should be for ByVal argument."); 7090 7091 const CCValAssign RL = ArgLocs[I++]; 7092 HandleRegLoc(RL.getLocReg(), Offset); 7093 FuncInfo->appendParameterType(PPCFunctionInfo::FixedType); 7094 } 7095 7096 if (Offset != StackSize) { 7097 assert(ArgLocs[I].getValNo() == VA.getValNo() && 7098 "Expected MemLoc for remaining bytes."); 7099 assert(ArgLocs[I].isMemLoc() && "Expected MemLoc for remaining bytes."); 7100 // Consume the MemLoc.The InVal has already been emitted, so nothing 7101 // more needs to be done. 7102 ++I; 7103 } 7104 7105 continue; 7106 } 7107 7108 if (VA.isRegLoc() && !VA.needsCustom()) { 7109 MVT::SimpleValueType SVT = ValVT.SimpleTy; 7110 Register VReg = 7111 MF.addLiveIn(VA.getLocReg(), 7112 getRegClassForSVT(SVT, IsPPC64, Subtarget.hasP8Vector(), 7113 Subtarget.hasVSX())); 7114 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl, VReg, LocVT); 7115 if (ValVT.isScalarInteger() && 7116 (ValVT.getFixedSizeInBits() < LocVT.getFixedSizeInBits())) { 7117 ArgValue = 7118 truncateScalarIntegerArg(Flags, ValVT, DAG, ArgValue, LocVT, dl); 7119 } 7120 InVals.push_back(ArgValue); 7121 continue; 7122 } 7123 if (VA.isMemLoc()) { 7124 HandleMemLoc(); 7125 continue; 7126 } 7127 } 7128 7129 // On AIX a minimum of 8 words is saved to the parameter save area. 7130 const unsigned MinParameterSaveArea = 8 * PtrByteSize; 7131 // Area that is at least reserved in the caller of this function. 7132 unsigned CallerReservedArea = 7133 std::max(CCInfo.getNextStackOffset(), LinkageSize + MinParameterSaveArea); 7134 7135 // Set the size that is at least reserved in caller of this function. Tail 7136 // call optimized function's reserved stack space needs to be aligned so 7137 // that taking the difference between two stack areas will result in an 7138 // aligned stack. 7139 CallerReservedArea = 7140 EnsureStackAlignment(Subtarget.getFrameLowering(), CallerReservedArea); 7141 FuncInfo->setMinReservedArea(CallerReservedArea); 7142 7143 if (isVarArg) { 7144 FuncInfo->setVarArgsFrameIndex( 7145 MFI.CreateFixedObject(PtrByteSize, CCInfo.getNextStackOffset(), true)); 7146 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 7147 7148 static const MCPhysReg GPR_32[] = {PPC::R3, PPC::R4, PPC::R5, PPC::R6, 7149 PPC::R7, PPC::R8, PPC::R9, PPC::R10}; 7150 7151 static const MCPhysReg GPR_64[] = {PPC::X3, PPC::X4, PPC::X5, PPC::X6, 7152 PPC::X7, PPC::X8, PPC::X9, PPC::X10}; 7153 const unsigned NumGPArgRegs = array_lengthof(IsPPC64 ? GPR_64 : GPR_32); 7154 7155 // The fixed integer arguments of a variadic function are stored to the 7156 // VarArgsFrameIndex on the stack so that they may be loaded by 7157 // dereferencing the result of va_next. 7158 for (unsigned GPRIndex = 7159 (CCInfo.getNextStackOffset() - LinkageSize) / PtrByteSize; 7160 GPRIndex < NumGPArgRegs; ++GPRIndex) { 7161 7162 const Register VReg = 7163 IsPPC64 ? MF.addLiveIn(GPR_64[GPRIndex], &PPC::G8RCRegClass) 7164 : MF.addLiveIn(GPR_32[GPRIndex], &PPC::GPRCRegClass); 7165 7166 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 7167 SDValue Store = 7168 DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo()); 7169 MemOps.push_back(Store); 7170 // Increment the address for the next argument to store. 7171 SDValue PtrOff = DAG.getConstant(PtrByteSize, dl, PtrVT); 7172 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); 7173 } 7174 } 7175 7176 if (!MemOps.empty()) 7177 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); 7178 7179 return Chain; 7180 } 7181 7182 SDValue PPCTargetLowering::LowerCall_AIX( 7183 SDValue Chain, SDValue Callee, CallFlags CFlags, 7184 const SmallVectorImpl<ISD::OutputArg> &Outs, 7185 const SmallVectorImpl<SDValue> &OutVals, 7186 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 7187 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, 7188 const CallBase *CB) const { 7189 // See PPCTargetLowering::LowerFormalArguments_AIX() for a description of the 7190 // AIX ABI stack frame layout. 7191 7192 assert((CFlags.CallConv == CallingConv::C || 7193 CFlags.CallConv == CallingConv::Cold || 7194 CFlags.CallConv == CallingConv::Fast) && 7195 "Unexpected calling convention!"); 7196 7197 if (CFlags.IsPatchPoint) 7198 report_fatal_error("This call type is unimplemented on AIX."); 7199 7200 const PPCSubtarget& Subtarget = 7201 static_cast<const PPCSubtarget&>(DAG.getSubtarget()); 7202 7203 MachineFunction &MF = DAG.getMachineFunction(); 7204 SmallVector<CCValAssign, 16> ArgLocs; 7205 AIXCCState CCInfo(CFlags.CallConv, CFlags.IsVarArg, MF, ArgLocs, 7206 *DAG.getContext()); 7207 7208 // Reserve space for the linkage save area (LSA) on the stack. 7209 // In both PPC32 and PPC64 there are 6 reserved slots in the LSA: 7210 // [SP][CR][LR][2 x reserved][TOC]. 7211 // The LSA is 24 bytes (6x4) in PPC32 and 48 bytes (6x8) in PPC64. 7212 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 7213 const bool IsPPC64 = Subtarget.isPPC64(); 7214 const EVT PtrVT = getPointerTy(DAG.getDataLayout()); 7215 const unsigned PtrByteSize = IsPPC64 ? 8 : 4; 7216 CCInfo.AllocateStack(LinkageSize, Align(PtrByteSize)); 7217 CCInfo.AnalyzeCallOperands(Outs, CC_AIX); 7218 7219 // The prolog code of the callee may store up to 8 GPR argument registers to 7220 // the stack, allowing va_start to index over them in memory if the callee 7221 // is variadic. 7222 // Because we cannot tell if this is needed on the caller side, we have to 7223 // conservatively assume that it is needed. As such, make sure we have at 7224 // least enough stack space for the caller to store the 8 GPRs. 7225 const unsigned MinParameterSaveAreaSize = 8 * PtrByteSize; 7226 const unsigned NumBytes = std::max(LinkageSize + MinParameterSaveAreaSize, 7227 CCInfo.getNextStackOffset()); 7228 7229 // Adjust the stack pointer for the new arguments... 7230 // These operations are automatically eliminated by the prolog/epilog pass. 7231 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); 7232 SDValue CallSeqStart = Chain; 7233 7234 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 7235 SmallVector<SDValue, 8> MemOpChains; 7236 7237 // Set up a copy of the stack pointer for loading and storing any 7238 // arguments that may not fit in the registers available for argument 7239 // passing. 7240 const SDValue StackPtr = IsPPC64 ? DAG.getRegister(PPC::X1, MVT::i64) 7241 : DAG.getRegister(PPC::R1, MVT::i32); 7242 7243 for (unsigned I = 0, E = ArgLocs.size(); I != E;) { 7244 const unsigned ValNo = ArgLocs[I].getValNo(); 7245 SDValue Arg = OutVals[ValNo]; 7246 ISD::ArgFlagsTy Flags = Outs[ValNo].Flags; 7247 7248 if (Flags.isByVal()) { 7249 const unsigned ByValSize = Flags.getByValSize(); 7250 7251 // Nothing to do for zero-sized ByVals on the caller side. 7252 if (!ByValSize) { 7253 ++I; 7254 continue; 7255 } 7256 7257 auto GetLoad = [&](EVT VT, unsigned LoadOffset) { 7258 return DAG.getExtLoad( 7259 ISD::ZEXTLOAD, dl, PtrVT, Chain, 7260 (LoadOffset != 0) 7261 ? DAG.getObjectPtrOffset(dl, Arg, TypeSize::Fixed(LoadOffset)) 7262 : Arg, 7263 MachinePointerInfo(), VT); 7264 }; 7265 7266 unsigned LoadOffset = 0; 7267 7268 // Initialize registers, which are fully occupied by the by-val argument. 7269 while (LoadOffset + PtrByteSize <= ByValSize && ArgLocs[I].isRegLoc()) { 7270 SDValue Load = GetLoad(PtrVT, LoadOffset); 7271 MemOpChains.push_back(Load.getValue(1)); 7272 LoadOffset += PtrByteSize; 7273 const CCValAssign &ByValVA = ArgLocs[I++]; 7274 assert(ByValVA.getValNo() == ValNo && 7275 "Unexpected location for pass-by-value argument."); 7276 RegsToPass.push_back(std::make_pair(ByValVA.getLocReg(), Load)); 7277 } 7278 7279 if (LoadOffset == ByValSize) 7280 continue; 7281 7282 // There must be one more loc to handle the remainder. 7283 assert(ArgLocs[I].getValNo() == ValNo && 7284 "Expected additional location for by-value argument."); 7285 7286 if (ArgLocs[I].isMemLoc()) { 7287 assert(LoadOffset < ByValSize && "Unexpected memloc for by-val arg."); 7288 const CCValAssign &ByValVA = ArgLocs[I++]; 7289 ISD::ArgFlagsTy MemcpyFlags = Flags; 7290 // Only memcpy the bytes that don't pass in register. 7291 MemcpyFlags.setByValSize(ByValSize - LoadOffset); 7292 Chain = CallSeqStart = createMemcpyOutsideCallSeq( 7293 (LoadOffset != 0) 7294 ? DAG.getObjectPtrOffset(dl, Arg, TypeSize::Fixed(LoadOffset)) 7295 : Arg, 7296 DAG.getObjectPtrOffset(dl, StackPtr, 7297 TypeSize::Fixed(ByValVA.getLocMemOffset())), 7298 CallSeqStart, MemcpyFlags, DAG, dl); 7299 continue; 7300 } 7301 7302 // Initialize the final register residue. 7303 // Any residue that occupies the final by-val arg register must be 7304 // left-justified on AIX. Loads must be a power-of-2 size and cannot be 7305 // larger than the ByValSize. For example: a 7 byte by-val arg requires 4, 7306 // 2 and 1 byte loads. 7307 const unsigned ResidueBytes = ByValSize % PtrByteSize; 7308 assert(ResidueBytes != 0 && LoadOffset + PtrByteSize > ByValSize && 7309 "Unexpected register residue for by-value argument."); 7310 SDValue ResidueVal; 7311 for (unsigned Bytes = 0; Bytes != ResidueBytes;) { 7312 const unsigned N = PowerOf2Floor(ResidueBytes - Bytes); 7313 const MVT VT = 7314 N == 1 ? MVT::i8 7315 : ((N == 2) ? MVT::i16 : (N == 4 ? MVT::i32 : MVT::i64)); 7316 SDValue Load = GetLoad(VT, LoadOffset); 7317 MemOpChains.push_back(Load.getValue(1)); 7318 LoadOffset += N; 7319 Bytes += N; 7320 7321 // By-val arguments are passed left-justfied in register. 7322 // Every load here needs to be shifted, otherwise a full register load 7323 // should have been used. 7324 assert(PtrVT.getSimpleVT().getSizeInBits() > (Bytes * 8) && 7325 "Unexpected load emitted during handling of pass-by-value " 7326 "argument."); 7327 unsigned NumSHLBits = PtrVT.getSimpleVT().getSizeInBits() - (Bytes * 8); 7328 EVT ShiftAmountTy = 7329 getShiftAmountTy(Load->getValueType(0), DAG.getDataLayout()); 7330 SDValue SHLAmt = DAG.getConstant(NumSHLBits, dl, ShiftAmountTy); 7331 SDValue ShiftedLoad = 7332 DAG.getNode(ISD::SHL, dl, Load.getValueType(), Load, SHLAmt); 7333 ResidueVal = ResidueVal ? DAG.getNode(ISD::OR, dl, PtrVT, ResidueVal, 7334 ShiftedLoad) 7335 : ShiftedLoad; 7336 } 7337 7338 const CCValAssign &ByValVA = ArgLocs[I++]; 7339 RegsToPass.push_back(std::make_pair(ByValVA.getLocReg(), ResidueVal)); 7340 continue; 7341 } 7342 7343 CCValAssign &VA = ArgLocs[I++]; 7344 const MVT LocVT = VA.getLocVT(); 7345 const MVT ValVT = VA.getValVT(); 7346 7347 switch (VA.getLocInfo()) { 7348 default: 7349 report_fatal_error("Unexpected argument extension type."); 7350 case CCValAssign::Full: 7351 break; 7352 case CCValAssign::ZExt: 7353 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg); 7354 break; 7355 case CCValAssign::SExt: 7356 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg); 7357 break; 7358 } 7359 7360 if (VA.isRegLoc() && !VA.needsCustom()) { 7361 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 7362 continue; 7363 } 7364 7365 // Vector arguments passed to VarArg functions need custom handling when 7366 // they are passed (at least partially) in GPRs. 7367 if (VA.isMemLoc() && VA.needsCustom() && ValVT.isVector()) { 7368 assert(CFlags.IsVarArg && "Custom MemLocs only used for Vector args."); 7369 // Store value to its stack slot. 7370 SDValue PtrOff = 7371 DAG.getConstant(VA.getLocMemOffset(), dl, StackPtr.getValueType()); 7372 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); 7373 SDValue Store = 7374 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()); 7375 MemOpChains.push_back(Store); 7376 const unsigned OriginalValNo = VA.getValNo(); 7377 // Then load the GPRs from the stack 7378 unsigned LoadOffset = 0; 7379 auto HandleCustomVecRegLoc = [&]() { 7380 assert(I != E && "Unexpected end of CCvalAssigns."); 7381 assert(ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() && 7382 "Expected custom RegLoc."); 7383 CCValAssign RegVA = ArgLocs[I++]; 7384 assert(RegVA.getValNo() == OriginalValNo && 7385 "Custom MemLoc ValNo and custom RegLoc ValNo must match."); 7386 SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, 7387 DAG.getConstant(LoadOffset, dl, PtrVT)); 7388 SDValue Load = DAG.getLoad(PtrVT, dl, Store, Add, MachinePointerInfo()); 7389 MemOpChains.push_back(Load.getValue(1)); 7390 RegsToPass.push_back(std::make_pair(RegVA.getLocReg(), Load)); 7391 LoadOffset += PtrByteSize; 7392 }; 7393 7394 // In 64-bit there will be exactly 2 custom RegLocs that follow, and in 7395 // in 32-bit there will be 2 custom RegLocs if we are passing in R9 and 7396 // R10. 7397 HandleCustomVecRegLoc(); 7398 HandleCustomVecRegLoc(); 7399 7400 if (I != E && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() && 7401 ArgLocs[I].getValNo() == OriginalValNo) { 7402 assert(!IsPPC64 && 7403 "Only 2 custom RegLocs expected for 64-bit codegen."); 7404 HandleCustomVecRegLoc(); 7405 HandleCustomVecRegLoc(); 7406 } 7407 7408 continue; 7409 } 7410 7411 if (VA.isMemLoc()) { 7412 SDValue PtrOff = 7413 DAG.getConstant(VA.getLocMemOffset(), dl, StackPtr.getValueType()); 7414 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); 7415 MemOpChains.push_back( 7416 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo())); 7417 7418 continue; 7419 } 7420 7421 if (!ValVT.isFloatingPoint()) 7422 report_fatal_error( 7423 "Unexpected register handling for calling convention."); 7424 7425 // Custom handling is used for GPR initializations for vararg float 7426 // arguments. 7427 assert(VA.isRegLoc() && VA.needsCustom() && CFlags.IsVarArg && 7428 LocVT.isInteger() && 7429 "Custom register handling only expected for VarArg."); 7430 7431 SDValue ArgAsInt = 7432 DAG.getBitcast(MVT::getIntegerVT(ValVT.getSizeInBits()), Arg); 7433 7434 if (Arg.getValueType().getStoreSize() == LocVT.getStoreSize()) 7435 // f32 in 32-bit GPR 7436 // f64 in 64-bit GPR 7437 RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgAsInt)); 7438 else if (Arg.getValueType().getFixedSizeInBits() < 7439 LocVT.getFixedSizeInBits()) 7440 // f32 in 64-bit GPR. 7441 RegsToPass.push_back(std::make_pair( 7442 VA.getLocReg(), DAG.getZExtOrTrunc(ArgAsInt, dl, LocVT))); 7443 else { 7444 // f64 in two 32-bit GPRs 7445 // The 2 GPRs are marked custom and expected to be adjacent in ArgLocs. 7446 assert(Arg.getValueType() == MVT::f64 && CFlags.IsVarArg && !IsPPC64 && 7447 "Unexpected custom register for argument!"); 7448 CCValAssign &GPR1 = VA; 7449 SDValue MSWAsI64 = DAG.getNode(ISD::SRL, dl, MVT::i64, ArgAsInt, 7450 DAG.getConstant(32, dl, MVT::i8)); 7451 RegsToPass.push_back(std::make_pair( 7452 GPR1.getLocReg(), DAG.getZExtOrTrunc(MSWAsI64, dl, MVT::i32))); 7453 7454 if (I != E) { 7455 // If only 1 GPR was available, there will only be one custom GPR and 7456 // the argument will also pass in memory. 7457 CCValAssign &PeekArg = ArgLocs[I]; 7458 if (PeekArg.isRegLoc() && PeekArg.getValNo() == PeekArg.getValNo()) { 7459 assert(PeekArg.needsCustom() && "A second custom GPR is expected."); 7460 CCValAssign &GPR2 = ArgLocs[I++]; 7461 RegsToPass.push_back(std::make_pair( 7462 GPR2.getLocReg(), DAG.getZExtOrTrunc(ArgAsInt, dl, MVT::i32))); 7463 } 7464 } 7465 } 7466 } 7467 7468 if (!MemOpChains.empty()) 7469 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); 7470 7471 // For indirect calls, we need to save the TOC base to the stack for 7472 // restoration after the call. 7473 if (CFlags.IsIndirect) { 7474 assert(!CFlags.IsTailCall && "Indirect tail-calls not supported."); 7475 const MCRegister TOCBaseReg = Subtarget.getTOCPointerRegister(); 7476 const MCRegister StackPtrReg = Subtarget.getStackPointerRegister(); 7477 const MVT PtrVT = Subtarget.isPPC64() ? MVT::i64 : MVT::i32; 7478 const unsigned TOCSaveOffset = 7479 Subtarget.getFrameLowering()->getTOCSaveOffset(); 7480 7481 setUsesTOCBasePtr(DAG); 7482 SDValue Val = DAG.getCopyFromReg(Chain, dl, TOCBaseReg, PtrVT); 7483 SDValue PtrOff = DAG.getIntPtrConstant(TOCSaveOffset, dl); 7484 SDValue StackPtr = DAG.getRegister(StackPtrReg, PtrVT); 7485 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); 7486 Chain = DAG.getStore( 7487 Val.getValue(1), dl, Val, AddPtr, 7488 MachinePointerInfo::getStack(DAG.getMachineFunction(), TOCSaveOffset)); 7489 } 7490 7491 // Build a sequence of copy-to-reg nodes chained together with token chain 7492 // and flag operands which copy the outgoing args into the appropriate regs. 7493 SDValue InFlag; 7494 for (auto Reg : RegsToPass) { 7495 Chain = DAG.getCopyToReg(Chain, dl, Reg.first, Reg.second, InFlag); 7496 InFlag = Chain.getValue(1); 7497 } 7498 7499 const int SPDiff = 0; 7500 return FinishCall(CFlags, dl, DAG, RegsToPass, InFlag, Chain, CallSeqStart, 7501 Callee, SPDiff, NumBytes, Ins, InVals, CB); 7502 } 7503 7504 bool 7505 PPCTargetLowering::CanLowerReturn(CallingConv::ID CallConv, 7506 MachineFunction &MF, bool isVarArg, 7507 const SmallVectorImpl<ISD::OutputArg> &Outs, 7508 LLVMContext &Context) const { 7509 SmallVector<CCValAssign, 16> RVLocs; 7510 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context); 7511 return CCInfo.CheckReturn( 7512 Outs, (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold) 7513 ? RetCC_PPC_Cold 7514 : RetCC_PPC); 7515 } 7516 7517 SDValue 7518 PPCTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, 7519 bool isVarArg, 7520 const SmallVectorImpl<ISD::OutputArg> &Outs, 7521 const SmallVectorImpl<SDValue> &OutVals, 7522 const SDLoc &dl, SelectionDAG &DAG) const { 7523 SmallVector<CCValAssign, 16> RVLocs; 7524 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, 7525 *DAG.getContext()); 7526 CCInfo.AnalyzeReturn(Outs, 7527 (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold) 7528 ? RetCC_PPC_Cold 7529 : RetCC_PPC); 7530 7531 SDValue Flag; 7532 SmallVector<SDValue, 4> RetOps(1, Chain); 7533 7534 // Copy the result values into the output registers. 7535 for (unsigned i = 0, RealResIdx = 0; i != RVLocs.size(); ++i, ++RealResIdx) { 7536 CCValAssign &VA = RVLocs[i]; 7537 assert(VA.isRegLoc() && "Can only return in registers!"); 7538 7539 SDValue Arg = OutVals[RealResIdx]; 7540 7541 switch (VA.getLocInfo()) { 7542 default: llvm_unreachable("Unknown loc info!"); 7543 case CCValAssign::Full: break; 7544 case CCValAssign::AExt: 7545 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg); 7546 break; 7547 case CCValAssign::ZExt: 7548 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg); 7549 break; 7550 case CCValAssign::SExt: 7551 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg); 7552 break; 7553 } 7554 if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) { 7555 bool isLittleEndian = Subtarget.isLittleEndian(); 7556 // Legalize ret f64 -> ret 2 x i32. 7557 SDValue SVal = 7558 DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg, 7559 DAG.getIntPtrConstant(isLittleEndian ? 0 : 1, dl)); 7560 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), SVal, Flag); 7561 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); 7562 SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg, 7563 DAG.getIntPtrConstant(isLittleEndian ? 1 : 0, dl)); 7564 Flag = Chain.getValue(1); 7565 VA = RVLocs[++i]; // skip ahead to next loc 7566 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), SVal, Flag); 7567 } else 7568 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Arg, Flag); 7569 Flag = Chain.getValue(1); 7570 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); 7571 } 7572 7573 RetOps[0] = Chain; // Update chain. 7574 7575 // Add the flag if we have it. 7576 if (Flag.getNode()) 7577 RetOps.push_back(Flag); 7578 7579 return DAG.getNode(PPCISD::RET_FLAG, dl, MVT::Other, RetOps); 7580 } 7581 7582 SDValue 7583 PPCTargetLowering::LowerGET_DYNAMIC_AREA_OFFSET(SDValue Op, 7584 SelectionDAG &DAG) const { 7585 SDLoc dl(Op); 7586 7587 // Get the correct type for integers. 7588 EVT IntVT = Op.getValueType(); 7589 7590 // Get the inputs. 7591 SDValue Chain = Op.getOperand(0); 7592 SDValue FPSIdx = getFramePointerFrameIndex(DAG); 7593 // Build a DYNAREAOFFSET node. 7594 SDValue Ops[2] = {Chain, FPSIdx}; 7595 SDVTList VTs = DAG.getVTList(IntVT); 7596 return DAG.getNode(PPCISD::DYNAREAOFFSET, dl, VTs, Ops); 7597 } 7598 7599 SDValue PPCTargetLowering::LowerSTACKRESTORE(SDValue Op, 7600 SelectionDAG &DAG) const { 7601 // When we pop the dynamic allocation we need to restore the SP link. 7602 SDLoc dl(Op); 7603 7604 // Get the correct type for pointers. 7605 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 7606 7607 // Construct the stack pointer operand. 7608 bool isPPC64 = Subtarget.isPPC64(); 7609 unsigned SP = isPPC64 ? PPC::X1 : PPC::R1; 7610 SDValue StackPtr = DAG.getRegister(SP, PtrVT); 7611 7612 // Get the operands for the STACKRESTORE. 7613 SDValue Chain = Op.getOperand(0); 7614 SDValue SaveSP = Op.getOperand(1); 7615 7616 // Load the old link SP. 7617 SDValue LoadLinkSP = 7618 DAG.getLoad(PtrVT, dl, Chain, StackPtr, MachinePointerInfo()); 7619 7620 // Restore the stack pointer. 7621 Chain = DAG.getCopyToReg(LoadLinkSP.getValue(1), dl, SP, SaveSP); 7622 7623 // Store the old link SP. 7624 return DAG.getStore(Chain, dl, LoadLinkSP, StackPtr, MachinePointerInfo()); 7625 } 7626 7627 SDValue PPCTargetLowering::getReturnAddrFrameIndex(SelectionDAG &DAG) const { 7628 MachineFunction &MF = DAG.getMachineFunction(); 7629 bool isPPC64 = Subtarget.isPPC64(); 7630 EVT PtrVT = getPointerTy(MF.getDataLayout()); 7631 7632 // Get current frame pointer save index. The users of this index will be 7633 // primarily DYNALLOC instructions. 7634 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>(); 7635 int RASI = FI->getReturnAddrSaveIndex(); 7636 7637 // If the frame pointer save index hasn't been defined yet. 7638 if (!RASI) { 7639 // Find out what the fix offset of the frame pointer save area. 7640 int LROffset = Subtarget.getFrameLowering()->getReturnSaveOffset(); 7641 // Allocate the frame index for frame pointer save area. 7642 RASI = MF.getFrameInfo().CreateFixedObject(isPPC64? 8 : 4, LROffset, false); 7643 // Save the result. 7644 FI->setReturnAddrSaveIndex(RASI); 7645 } 7646 return DAG.getFrameIndex(RASI, PtrVT); 7647 } 7648 7649 SDValue 7650 PPCTargetLowering::getFramePointerFrameIndex(SelectionDAG & DAG) const { 7651 MachineFunction &MF = DAG.getMachineFunction(); 7652 bool isPPC64 = Subtarget.isPPC64(); 7653 EVT PtrVT = getPointerTy(MF.getDataLayout()); 7654 7655 // Get current frame pointer save index. The users of this index will be 7656 // primarily DYNALLOC instructions. 7657 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>(); 7658 int FPSI = FI->getFramePointerSaveIndex(); 7659 7660 // If the frame pointer save index hasn't been defined yet. 7661 if (!FPSI) { 7662 // Find out what the fix offset of the frame pointer save area. 7663 int FPOffset = Subtarget.getFrameLowering()->getFramePointerSaveOffset(); 7664 // Allocate the frame index for frame pointer save area. 7665 FPSI = MF.getFrameInfo().CreateFixedObject(isPPC64? 8 : 4, FPOffset, true); 7666 // Save the result. 7667 FI->setFramePointerSaveIndex(FPSI); 7668 } 7669 return DAG.getFrameIndex(FPSI, PtrVT); 7670 } 7671 7672 SDValue PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, 7673 SelectionDAG &DAG) const { 7674 MachineFunction &MF = DAG.getMachineFunction(); 7675 // Get the inputs. 7676 SDValue Chain = Op.getOperand(0); 7677 SDValue Size = Op.getOperand(1); 7678 SDLoc dl(Op); 7679 7680 // Get the correct type for pointers. 7681 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 7682 // Negate the size. 7683 SDValue NegSize = DAG.getNode(ISD::SUB, dl, PtrVT, 7684 DAG.getConstant(0, dl, PtrVT), Size); 7685 // Construct a node for the frame pointer save index. 7686 SDValue FPSIdx = getFramePointerFrameIndex(DAG); 7687 SDValue Ops[3] = { Chain, NegSize, FPSIdx }; 7688 SDVTList VTs = DAG.getVTList(PtrVT, MVT::Other); 7689 if (hasInlineStackProbe(MF)) 7690 return DAG.getNode(PPCISD::PROBED_ALLOCA, dl, VTs, Ops); 7691 return DAG.getNode(PPCISD::DYNALLOC, dl, VTs, Ops); 7692 } 7693 7694 SDValue PPCTargetLowering::LowerEH_DWARF_CFA(SDValue Op, 7695 SelectionDAG &DAG) const { 7696 MachineFunction &MF = DAG.getMachineFunction(); 7697 7698 bool isPPC64 = Subtarget.isPPC64(); 7699 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 7700 7701 int FI = MF.getFrameInfo().CreateFixedObject(isPPC64 ? 8 : 4, 0, false); 7702 return DAG.getFrameIndex(FI, PtrVT); 7703 } 7704 7705 SDValue PPCTargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op, 7706 SelectionDAG &DAG) const { 7707 SDLoc DL(Op); 7708 return DAG.getNode(PPCISD::EH_SJLJ_SETJMP, DL, 7709 DAG.getVTList(MVT::i32, MVT::Other), 7710 Op.getOperand(0), Op.getOperand(1)); 7711 } 7712 7713 SDValue PPCTargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op, 7714 SelectionDAG &DAG) const { 7715 SDLoc DL(Op); 7716 return DAG.getNode(PPCISD::EH_SJLJ_LONGJMP, DL, MVT::Other, 7717 Op.getOperand(0), Op.getOperand(1)); 7718 } 7719 7720 SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const { 7721 if (Op.getValueType().isVector()) 7722 return LowerVectorLoad(Op, DAG); 7723 7724 assert(Op.getValueType() == MVT::i1 && 7725 "Custom lowering only for i1 loads"); 7726 7727 // First, load 8 bits into 32 bits, then truncate to 1 bit. 7728 7729 SDLoc dl(Op); 7730 LoadSDNode *LD = cast<LoadSDNode>(Op); 7731 7732 SDValue Chain = LD->getChain(); 7733 SDValue BasePtr = LD->getBasePtr(); 7734 MachineMemOperand *MMO = LD->getMemOperand(); 7735 7736 SDValue NewLD = 7737 DAG.getExtLoad(ISD::EXTLOAD, dl, getPointerTy(DAG.getDataLayout()), Chain, 7738 BasePtr, MVT::i8, MMO); 7739 SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewLD); 7740 7741 SDValue Ops[] = { Result, SDValue(NewLD.getNode(), 1) }; 7742 return DAG.getMergeValues(Ops, dl); 7743 } 7744 7745 SDValue PPCTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const { 7746 if (Op.getOperand(1).getValueType().isVector()) 7747 return LowerVectorStore(Op, DAG); 7748 7749 assert(Op.getOperand(1).getValueType() == MVT::i1 && 7750 "Custom lowering only for i1 stores"); 7751 7752 // First, zero extend to 32 bits, then use a truncating store to 8 bits. 7753 7754 SDLoc dl(Op); 7755 StoreSDNode *ST = cast<StoreSDNode>(Op); 7756 7757 SDValue Chain = ST->getChain(); 7758 SDValue BasePtr = ST->getBasePtr(); 7759 SDValue Value = ST->getValue(); 7760 MachineMemOperand *MMO = ST->getMemOperand(); 7761 7762 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, getPointerTy(DAG.getDataLayout()), 7763 Value); 7764 return DAG.getTruncStore(Chain, dl, Value, BasePtr, MVT::i8, MMO); 7765 } 7766 7767 // FIXME: Remove this once the ANDI glue bug is fixed: 7768 SDValue PPCTargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const { 7769 assert(Op.getValueType() == MVT::i1 && 7770 "Custom lowering only for i1 results"); 7771 7772 SDLoc DL(Op); 7773 return DAG.getNode(PPCISD::ANDI_rec_1_GT_BIT, DL, MVT::i1, Op.getOperand(0)); 7774 } 7775 7776 SDValue PPCTargetLowering::LowerTRUNCATEVector(SDValue Op, 7777 SelectionDAG &DAG) const { 7778 7779 // Implements a vector truncate that fits in a vector register as a shuffle. 7780 // We want to legalize vector truncates down to where the source fits in 7781 // a vector register (and target is therefore smaller than vector register 7782 // size). At that point legalization will try to custom lower the sub-legal 7783 // result and get here - where we can contain the truncate as a single target 7784 // operation. 7785 7786 // For example a trunc <2 x i16> to <2 x i8> could be visualized as follows: 7787 // <MSB1|LSB1, MSB2|LSB2> to <LSB1, LSB2> 7788 // 7789 // We will implement it for big-endian ordering as this (where x denotes 7790 // undefined): 7791 // < MSB1|LSB1, MSB2|LSB2, uu, uu, uu, uu, uu, uu> to 7792 // < LSB1, LSB2, u, u, u, u, u, u, u, u, u, u, u, u, u, u> 7793 // 7794 // The same operation in little-endian ordering will be: 7795 // <uu, uu, uu, uu, uu, uu, LSB2|MSB2, LSB1|MSB1> to 7796 // <u, u, u, u, u, u, u, u, u, u, u, u, u, u, LSB2, LSB1> 7797 7798 EVT TrgVT = Op.getValueType(); 7799 assert(TrgVT.isVector() && "Vector type expected."); 7800 unsigned TrgNumElts = TrgVT.getVectorNumElements(); 7801 EVT EltVT = TrgVT.getVectorElementType(); 7802 if (!isOperationCustom(Op.getOpcode(), TrgVT) || 7803 TrgVT.getSizeInBits() > 128 || !isPowerOf2_32(TrgNumElts) || 7804 !isPowerOf2_32(EltVT.getSizeInBits())) 7805 return SDValue(); 7806 7807 SDValue N1 = Op.getOperand(0); 7808 EVT SrcVT = N1.getValueType(); 7809 unsigned SrcSize = SrcVT.getSizeInBits(); 7810 if (SrcSize > 256 || 7811 !isPowerOf2_32(SrcVT.getVectorNumElements()) || 7812 !isPowerOf2_32(SrcVT.getVectorElementType().getSizeInBits())) 7813 return SDValue(); 7814 if (SrcSize == 256 && SrcVT.getVectorNumElements() < 2) 7815 return SDValue(); 7816 7817 unsigned WideNumElts = 128 / EltVT.getSizeInBits(); 7818 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT, WideNumElts); 7819 7820 SDLoc DL(Op); 7821 SDValue Op1, Op2; 7822 if (SrcSize == 256) { 7823 EVT VecIdxTy = getVectorIdxTy(DAG.getDataLayout()); 7824 EVT SplitVT = 7825 N1.getValueType().getHalfNumVectorElementsVT(*DAG.getContext()); 7826 unsigned SplitNumElts = SplitVT.getVectorNumElements(); 7827 Op1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, N1, 7828 DAG.getConstant(0, DL, VecIdxTy)); 7829 Op2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, N1, 7830 DAG.getConstant(SplitNumElts, DL, VecIdxTy)); 7831 } 7832 else { 7833 Op1 = SrcSize == 128 ? N1 : widenVec(DAG, N1, DL); 7834 Op2 = DAG.getUNDEF(WideVT); 7835 } 7836 7837 // First list the elements we want to keep. 7838 unsigned SizeMult = SrcSize / TrgVT.getSizeInBits(); 7839 SmallVector<int, 16> ShuffV; 7840 if (Subtarget.isLittleEndian()) 7841 for (unsigned i = 0; i < TrgNumElts; ++i) 7842 ShuffV.push_back(i * SizeMult); 7843 else 7844 for (unsigned i = 1; i <= TrgNumElts; ++i) 7845 ShuffV.push_back(i * SizeMult - 1); 7846 7847 // Populate the remaining elements with undefs. 7848 for (unsigned i = TrgNumElts; i < WideNumElts; ++i) 7849 // ShuffV.push_back(i + WideNumElts); 7850 ShuffV.push_back(WideNumElts + 1); 7851 7852 Op1 = DAG.getNode(ISD::BITCAST, DL, WideVT, Op1); 7853 Op2 = DAG.getNode(ISD::BITCAST, DL, WideVT, Op2); 7854 return DAG.getVectorShuffle(WideVT, DL, Op1, Op2, ShuffV); 7855 } 7856 7857 /// LowerSELECT_CC - Lower floating point select_cc's into fsel instruction when 7858 /// possible. 7859 SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { 7860 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 7861 EVT ResVT = Op.getValueType(); 7862 EVT CmpVT = Op.getOperand(0).getValueType(); 7863 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); 7864 SDValue TV = Op.getOperand(2), FV = Op.getOperand(3); 7865 SDLoc dl(Op); 7866 7867 // Without power9-vector, we don't have native instruction for f128 comparison. 7868 // Following transformation to libcall is needed for setcc: 7869 // select_cc lhs, rhs, tv, fv, cc -> select_cc (setcc cc, x, y), 0, tv, fv, NE 7870 if (!Subtarget.hasP9Vector() && CmpVT == MVT::f128) { 7871 SDValue Z = DAG.getSetCC( 7872 dl, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT), 7873 LHS, RHS, CC); 7874 SDValue Zero = DAG.getConstant(0, dl, Z.getValueType()); 7875 return DAG.getSelectCC(dl, Z, Zero, TV, FV, ISD::SETNE); 7876 } 7877 7878 // Not FP, or using SPE? Not a fsel. 7879 if (!CmpVT.isFloatingPoint() || !TV.getValueType().isFloatingPoint() || 7880 Subtarget.hasSPE()) 7881 return Op; 7882 7883 SDNodeFlags Flags = Op.getNode()->getFlags(); 7884 7885 // We have xsmaxcdp/xsmincdp which are OK to emit even in the 7886 // presence of infinities. 7887 if (Subtarget.hasP9Vector() && LHS == TV && RHS == FV) { 7888 switch (CC) { 7889 default: 7890 break; 7891 case ISD::SETOGT: 7892 case ISD::SETGT: 7893 return DAG.getNode(PPCISD::XSMAXCDP, dl, Op.getValueType(), LHS, RHS); 7894 case ISD::SETOLT: 7895 case ISD::SETLT: 7896 return DAG.getNode(PPCISD::XSMINCDP, dl, Op.getValueType(), LHS, RHS); 7897 } 7898 } 7899 7900 // We might be able to do better than this under some circumstances, but in 7901 // general, fsel-based lowering of select is a finite-math-only optimization. 7902 // For more information, see section F.3 of the 2.06 ISA specification. 7903 // With ISA 3.0 7904 if ((!DAG.getTarget().Options.NoInfsFPMath && !Flags.hasNoInfs()) || 7905 (!DAG.getTarget().Options.NoNaNsFPMath && !Flags.hasNoNaNs())) 7906 return Op; 7907 7908 // If the RHS of the comparison is a 0.0, we don't need to do the 7909 // subtraction at all. 7910 SDValue Sel1; 7911 if (isFloatingPointZero(RHS)) 7912 switch (CC) { 7913 default: break; // SETUO etc aren't handled by fsel. 7914 case ISD::SETNE: 7915 std::swap(TV, FV); 7916 LLVM_FALLTHROUGH; 7917 case ISD::SETEQ: 7918 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits 7919 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); 7920 Sel1 = DAG.getNode(PPCISD::FSEL, dl, ResVT, LHS, TV, FV); 7921 if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits 7922 Sel1 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Sel1); 7923 return DAG.getNode(PPCISD::FSEL, dl, ResVT, 7924 DAG.getNode(ISD::FNEG, dl, MVT::f64, LHS), Sel1, FV); 7925 case ISD::SETULT: 7926 case ISD::SETLT: 7927 std::swap(TV, FV); // fsel is natively setge, swap operands for setlt 7928 LLVM_FALLTHROUGH; 7929 case ISD::SETOGE: 7930 case ISD::SETGE: 7931 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits 7932 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); 7933 return DAG.getNode(PPCISD::FSEL, dl, ResVT, LHS, TV, FV); 7934 case ISD::SETUGT: 7935 case ISD::SETGT: 7936 std::swap(TV, FV); // fsel is natively setge, swap operands for setlt 7937 LLVM_FALLTHROUGH; 7938 case ISD::SETOLE: 7939 case ISD::SETLE: 7940 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits 7941 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); 7942 return DAG.getNode(PPCISD::FSEL, dl, ResVT, 7943 DAG.getNode(ISD::FNEG, dl, MVT::f64, LHS), TV, FV); 7944 } 7945 7946 SDValue Cmp; 7947 switch (CC) { 7948 default: break; // SETUO etc aren't handled by fsel. 7949 case ISD::SETNE: 7950 std::swap(TV, FV); 7951 LLVM_FALLTHROUGH; 7952 case ISD::SETEQ: 7953 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags); 7954 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 7955 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 7956 Sel1 = DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV); 7957 if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits 7958 Sel1 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Sel1); 7959 return DAG.getNode(PPCISD::FSEL, dl, ResVT, 7960 DAG.getNode(ISD::FNEG, dl, MVT::f64, Cmp), Sel1, FV); 7961 case ISD::SETULT: 7962 case ISD::SETLT: 7963 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags); 7964 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 7965 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 7966 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, FV, TV); 7967 case ISD::SETOGE: 7968 case ISD::SETGE: 7969 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags); 7970 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 7971 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 7972 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV); 7973 case ISD::SETUGT: 7974 case ISD::SETGT: 7975 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, RHS, LHS, Flags); 7976 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 7977 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 7978 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, FV, TV); 7979 case ISD::SETOLE: 7980 case ISD::SETLE: 7981 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, RHS, LHS, Flags); 7982 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 7983 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 7984 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV); 7985 } 7986 return Op; 7987 } 7988 7989 static unsigned getPPCStrictOpcode(unsigned Opc) { 7990 switch (Opc) { 7991 default: 7992 llvm_unreachable("No strict version of this opcode!"); 7993 case PPCISD::FCTIDZ: 7994 return PPCISD::STRICT_FCTIDZ; 7995 case PPCISD::FCTIWZ: 7996 return PPCISD::STRICT_FCTIWZ; 7997 case PPCISD::FCTIDUZ: 7998 return PPCISD::STRICT_FCTIDUZ; 7999 case PPCISD::FCTIWUZ: 8000 return PPCISD::STRICT_FCTIWUZ; 8001 case PPCISD::FCFID: 8002 return PPCISD::STRICT_FCFID; 8003 case PPCISD::FCFIDU: 8004 return PPCISD::STRICT_FCFIDU; 8005 case PPCISD::FCFIDS: 8006 return PPCISD::STRICT_FCFIDS; 8007 case PPCISD::FCFIDUS: 8008 return PPCISD::STRICT_FCFIDUS; 8009 } 8010 } 8011 8012 static SDValue convertFPToInt(SDValue Op, SelectionDAG &DAG, 8013 const PPCSubtarget &Subtarget) { 8014 SDLoc dl(Op); 8015 bool IsStrict = Op->isStrictFPOpcode(); 8016 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT || 8017 Op.getOpcode() == ISD::STRICT_FP_TO_SINT; 8018 8019 // TODO: Any other flags to propagate? 8020 SDNodeFlags Flags; 8021 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept()); 8022 8023 // For strict nodes, source is the second operand. 8024 SDValue Src = Op.getOperand(IsStrict ? 1 : 0); 8025 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue(); 8026 assert(Src.getValueType().isFloatingPoint()); 8027 if (Src.getValueType() == MVT::f32) { 8028 if (IsStrict) { 8029 Src = 8030 DAG.getNode(ISD::STRICT_FP_EXTEND, dl, 8031 DAG.getVTList(MVT::f64, MVT::Other), {Chain, Src}, Flags); 8032 Chain = Src.getValue(1); 8033 } else 8034 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src); 8035 } 8036 SDValue Conv; 8037 unsigned Opc = ISD::DELETED_NODE; 8038 switch (Op.getSimpleValueType().SimpleTy) { 8039 default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!"); 8040 case MVT::i32: 8041 Opc = IsSigned ? PPCISD::FCTIWZ 8042 : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ); 8043 break; 8044 case MVT::i64: 8045 assert((IsSigned || Subtarget.hasFPCVT()) && 8046 "i64 FP_TO_UINT is supported only with FPCVT"); 8047 Opc = IsSigned ? PPCISD::FCTIDZ : PPCISD::FCTIDUZ; 8048 } 8049 if (IsStrict) { 8050 Opc = getPPCStrictOpcode(Opc); 8051 Conv = DAG.getNode(Opc, dl, DAG.getVTList(MVT::f64, MVT::Other), 8052 {Chain, Src}, Flags); 8053 } else { 8054 Conv = DAG.getNode(Opc, dl, MVT::f64, Src); 8055 } 8056 return Conv; 8057 } 8058 8059 void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI, 8060 SelectionDAG &DAG, 8061 const SDLoc &dl) const { 8062 SDValue Tmp = convertFPToInt(Op, DAG, Subtarget); 8063 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT || 8064 Op.getOpcode() == ISD::STRICT_FP_TO_SINT; 8065 bool IsStrict = Op->isStrictFPOpcode(); 8066 8067 // Convert the FP value to an int value through memory. 8068 bool i32Stack = Op.getValueType() == MVT::i32 && Subtarget.hasSTFIWX() && 8069 (IsSigned || Subtarget.hasFPCVT()); 8070 SDValue FIPtr = DAG.CreateStackTemporary(i32Stack ? MVT::i32 : MVT::f64); 8071 int FI = cast<FrameIndexSDNode>(FIPtr)->getIndex(); 8072 MachinePointerInfo MPI = 8073 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); 8074 8075 // Emit a store to the stack slot. 8076 SDValue Chain = IsStrict ? Tmp.getValue(1) : DAG.getEntryNode(); 8077 Align Alignment(DAG.getEVTAlign(Tmp.getValueType())); 8078 if (i32Stack) { 8079 MachineFunction &MF = DAG.getMachineFunction(); 8080 Alignment = Align(4); 8081 MachineMemOperand *MMO = 8082 MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, Alignment); 8083 SDValue Ops[] = { Chain, Tmp, FIPtr }; 8084 Chain = DAG.getMemIntrinsicNode(PPCISD::STFIWX, dl, 8085 DAG.getVTList(MVT::Other), Ops, MVT::i32, MMO); 8086 } else 8087 Chain = DAG.getStore(Chain, dl, Tmp, FIPtr, MPI, Alignment); 8088 8089 // Result is a load from the stack slot. If loading 4 bytes, make sure to 8090 // add in a bias on big endian. 8091 if (Op.getValueType() == MVT::i32 && !i32Stack) { 8092 FIPtr = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, 8093 DAG.getConstant(4, dl, FIPtr.getValueType())); 8094 MPI = MPI.getWithOffset(Subtarget.isLittleEndian() ? 0 : 4); 8095 } 8096 8097 RLI.Chain = Chain; 8098 RLI.Ptr = FIPtr; 8099 RLI.MPI = MPI; 8100 RLI.Alignment = Alignment; 8101 } 8102 8103 /// Custom lowers floating point to integer conversions to use 8104 /// the direct move instructions available in ISA 2.07 to avoid the 8105 /// need for load/store combinations. 8106 SDValue PPCTargetLowering::LowerFP_TO_INTDirectMove(SDValue Op, 8107 SelectionDAG &DAG, 8108 const SDLoc &dl) const { 8109 SDValue Conv = convertFPToInt(Op, DAG, Subtarget); 8110 SDValue Mov = DAG.getNode(PPCISD::MFVSR, dl, Op.getValueType(), Conv); 8111 if (Op->isStrictFPOpcode()) 8112 return DAG.getMergeValues({Mov, Conv.getValue(1)}, dl); 8113 else 8114 return Mov; 8115 } 8116 8117 SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG, 8118 const SDLoc &dl) const { 8119 bool IsStrict = Op->isStrictFPOpcode(); 8120 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT || 8121 Op.getOpcode() == ISD::STRICT_FP_TO_SINT; 8122 SDValue Src = Op.getOperand(IsStrict ? 1 : 0); 8123 EVT SrcVT = Src.getValueType(); 8124 EVT DstVT = Op.getValueType(); 8125 8126 // FP to INT conversions are legal for f128. 8127 if (SrcVT == MVT::f128) 8128 return Subtarget.hasP9Vector() ? Op : SDValue(); 8129 8130 // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on 8131 // PPC (the libcall is not available). 8132 if (SrcVT == MVT::ppcf128) { 8133 if (DstVT == MVT::i32) { 8134 // TODO: Conservatively pass only nofpexcept flag here. Need to check and 8135 // set other fast-math flags to FP operations in both strict and 8136 // non-strict cases. (FP_TO_SINT, FSUB) 8137 SDNodeFlags Flags; 8138 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept()); 8139 8140 if (IsSigned) { 8141 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::f64, Src, 8142 DAG.getIntPtrConstant(0, dl)); 8143 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::f64, Src, 8144 DAG.getIntPtrConstant(1, dl)); 8145 8146 // Add the two halves of the long double in round-to-zero mode, and use 8147 // a smaller FP_TO_SINT. 8148 if (IsStrict) { 8149 SDValue Res = DAG.getNode(PPCISD::STRICT_FADDRTZ, dl, 8150 DAG.getVTList(MVT::f64, MVT::Other), 8151 {Op.getOperand(0), Lo, Hi}, Flags); 8152 return DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, 8153 DAG.getVTList(MVT::i32, MVT::Other), 8154 {Res.getValue(1), Res}, Flags); 8155 } else { 8156 SDValue Res = DAG.getNode(PPCISD::FADDRTZ, dl, MVT::f64, Lo, Hi); 8157 return DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Res); 8158 } 8159 } else { 8160 const uint64_t TwoE31[] = {0x41e0000000000000LL, 0}; 8161 APFloat APF = APFloat(APFloat::PPCDoubleDouble(), APInt(128, TwoE31)); 8162 SDValue Cst = DAG.getConstantFP(APF, dl, SrcVT); 8163 SDValue SignMask = DAG.getConstant(0x80000000, dl, DstVT); 8164 if (IsStrict) { 8165 // Sel = Src < 0x80000000 8166 // FltOfs = select Sel, 0.0, 0x80000000 8167 // IntOfs = select Sel, 0, 0x80000000 8168 // Result = fp_to_sint(Src - FltOfs) ^ IntOfs 8169 SDValue Chain = Op.getOperand(0); 8170 EVT SetCCVT = 8171 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT); 8172 EVT DstSetCCVT = 8173 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), DstVT); 8174 SDValue Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT, 8175 Chain, true); 8176 Chain = Sel.getValue(1); 8177 8178 SDValue FltOfs = DAG.getSelect( 8179 dl, SrcVT, Sel, DAG.getConstantFP(0.0, dl, SrcVT), Cst); 8180 Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT); 8181 8182 SDValue Val = DAG.getNode(ISD::STRICT_FSUB, dl, 8183 DAG.getVTList(SrcVT, MVT::Other), 8184 {Chain, Src, FltOfs}, Flags); 8185 Chain = Val.getValue(1); 8186 SDValue SInt = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, 8187 DAG.getVTList(DstVT, MVT::Other), 8188 {Chain, Val}, Flags); 8189 Chain = SInt.getValue(1); 8190 SDValue IntOfs = DAG.getSelect( 8191 dl, DstVT, Sel, DAG.getConstant(0, dl, DstVT), SignMask); 8192 SDValue Result = DAG.getNode(ISD::XOR, dl, DstVT, SInt, IntOfs); 8193 return DAG.getMergeValues({Result, Chain}, dl); 8194 } else { 8195 // X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X 8196 // FIXME: generated code sucks. 8197 SDValue True = DAG.getNode(ISD::FSUB, dl, MVT::ppcf128, Src, Cst); 8198 True = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, True); 8199 True = DAG.getNode(ISD::ADD, dl, MVT::i32, True, SignMask); 8200 SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Src); 8201 return DAG.getSelectCC(dl, Src, Cst, True, False, ISD::SETGE); 8202 } 8203 } 8204 } 8205 8206 return SDValue(); 8207 } 8208 8209 if (Subtarget.hasDirectMove() && Subtarget.isPPC64()) 8210 return LowerFP_TO_INTDirectMove(Op, DAG, dl); 8211 8212 ReuseLoadInfo RLI; 8213 LowerFP_TO_INTForReuse(Op, RLI, DAG, dl); 8214 8215 return DAG.getLoad(Op.getValueType(), dl, RLI.Chain, RLI.Ptr, RLI.MPI, 8216 RLI.Alignment, RLI.MMOFlags(), RLI.AAInfo, RLI.Ranges); 8217 } 8218 8219 // We're trying to insert a regular store, S, and then a load, L. If the 8220 // incoming value, O, is a load, we might just be able to have our load use the 8221 // address used by O. However, we don't know if anything else will store to 8222 // that address before we can load from it. To prevent this situation, we need 8223 // to insert our load, L, into the chain as a peer of O. To do this, we give L 8224 // the same chain operand as O, we create a token factor from the chain results 8225 // of O and L, and we replace all uses of O's chain result with that token 8226 // factor (see spliceIntoChain below for this last part). 8227 bool PPCTargetLowering::canReuseLoadAddress(SDValue Op, EVT MemVT, 8228 ReuseLoadInfo &RLI, 8229 SelectionDAG &DAG, 8230 ISD::LoadExtType ET) const { 8231 // Conservatively skip reusing for constrained FP nodes. 8232 if (Op->isStrictFPOpcode()) 8233 return false; 8234 8235 SDLoc dl(Op); 8236 bool ValidFPToUint = Op.getOpcode() == ISD::FP_TO_UINT && 8237 (Subtarget.hasFPCVT() || Op.getValueType() == MVT::i32); 8238 if (ET == ISD::NON_EXTLOAD && 8239 (ValidFPToUint || Op.getOpcode() == ISD::FP_TO_SINT) && 8240 isOperationLegalOrCustom(Op.getOpcode(), 8241 Op.getOperand(0).getValueType())) { 8242 8243 LowerFP_TO_INTForReuse(Op, RLI, DAG, dl); 8244 return true; 8245 } 8246 8247 LoadSDNode *LD = dyn_cast<LoadSDNode>(Op); 8248 if (!LD || LD->getExtensionType() != ET || LD->isVolatile() || 8249 LD->isNonTemporal()) 8250 return false; 8251 if (LD->getMemoryVT() != MemVT) 8252 return false; 8253 8254 // If the result of the load is an illegal type, then we can't build a 8255 // valid chain for reuse since the legalised loads and token factor node that 8256 // ties the legalised loads together uses a different output chain then the 8257 // illegal load. 8258 if (!isTypeLegal(LD->getValueType(0))) 8259 return false; 8260 8261 RLI.Ptr = LD->getBasePtr(); 8262 if (LD->isIndexed() && !LD->getOffset().isUndef()) { 8263 assert(LD->getAddressingMode() == ISD::PRE_INC && 8264 "Non-pre-inc AM on PPC?"); 8265 RLI.Ptr = DAG.getNode(ISD::ADD, dl, RLI.Ptr.getValueType(), RLI.Ptr, 8266 LD->getOffset()); 8267 } 8268 8269 RLI.Chain = LD->getChain(); 8270 RLI.MPI = LD->getPointerInfo(); 8271 RLI.IsDereferenceable = LD->isDereferenceable(); 8272 RLI.IsInvariant = LD->isInvariant(); 8273 RLI.Alignment = LD->getAlign(); 8274 RLI.AAInfo = LD->getAAInfo(); 8275 RLI.Ranges = LD->getRanges(); 8276 8277 RLI.ResChain = SDValue(LD, LD->isIndexed() ? 2 : 1); 8278 return true; 8279 } 8280 8281 // Given the head of the old chain, ResChain, insert a token factor containing 8282 // it and NewResChain, and make users of ResChain now be users of that token 8283 // factor. 8284 // TODO: Remove and use DAG::makeEquivalentMemoryOrdering() instead. 8285 void PPCTargetLowering::spliceIntoChain(SDValue ResChain, 8286 SDValue NewResChain, 8287 SelectionDAG &DAG) const { 8288 if (!ResChain) 8289 return; 8290 8291 SDLoc dl(NewResChain); 8292 8293 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 8294 NewResChain, DAG.getUNDEF(MVT::Other)); 8295 assert(TF.getNode() != NewResChain.getNode() && 8296 "A new TF really is required here"); 8297 8298 DAG.ReplaceAllUsesOfValueWith(ResChain, TF); 8299 DAG.UpdateNodeOperands(TF.getNode(), ResChain, NewResChain); 8300 } 8301 8302 /// Analyze profitability of direct move 8303 /// prefer float load to int load plus direct move 8304 /// when there is no integer use of int load 8305 bool PPCTargetLowering::directMoveIsProfitable(const SDValue &Op) const { 8306 SDNode *Origin = Op.getOperand(0).getNode(); 8307 if (Origin->getOpcode() != ISD::LOAD) 8308 return true; 8309 8310 // If there is no LXSIBZX/LXSIHZX, like Power8, 8311 // prefer direct move if the memory size is 1 or 2 bytes. 8312 MachineMemOperand *MMO = cast<LoadSDNode>(Origin)->getMemOperand(); 8313 if (!Subtarget.hasP9Vector() && MMO->getSize() <= 2) 8314 return true; 8315 8316 for (SDNode::use_iterator UI = Origin->use_begin(), 8317 UE = Origin->use_end(); 8318 UI != UE; ++UI) { 8319 8320 // Only look at the users of the loaded value. 8321 if (UI.getUse().get().getResNo() != 0) 8322 continue; 8323 8324 if (UI->getOpcode() != ISD::SINT_TO_FP && 8325 UI->getOpcode() != ISD::UINT_TO_FP && 8326 UI->getOpcode() != ISD::STRICT_SINT_TO_FP && 8327 UI->getOpcode() != ISD::STRICT_UINT_TO_FP) 8328 return true; 8329 } 8330 8331 return false; 8332 } 8333 8334 static SDValue convertIntToFP(SDValue Op, SDValue Src, SelectionDAG &DAG, 8335 const PPCSubtarget &Subtarget, 8336 SDValue Chain = SDValue()) { 8337 bool IsSigned = Op.getOpcode() == ISD::SINT_TO_FP || 8338 Op.getOpcode() == ISD::STRICT_SINT_TO_FP; 8339 SDLoc dl(Op); 8340 8341 // TODO: Any other flags to propagate? 8342 SDNodeFlags Flags; 8343 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept()); 8344 8345 // If we have FCFIDS, then use it when converting to single-precision. 8346 // Otherwise, convert to double-precision and then round. 8347 bool IsSingle = Op.getValueType() == MVT::f32 && Subtarget.hasFPCVT(); 8348 unsigned ConvOpc = IsSingle ? (IsSigned ? PPCISD::FCFIDS : PPCISD::FCFIDUS) 8349 : (IsSigned ? PPCISD::FCFID : PPCISD::FCFIDU); 8350 EVT ConvTy = IsSingle ? MVT::f32 : MVT::f64; 8351 if (Op->isStrictFPOpcode()) { 8352 if (!Chain) 8353 Chain = Op.getOperand(0); 8354 return DAG.getNode(getPPCStrictOpcode(ConvOpc), dl, 8355 DAG.getVTList(ConvTy, MVT::Other), {Chain, Src}, Flags); 8356 } else 8357 return DAG.getNode(ConvOpc, dl, ConvTy, Src); 8358 } 8359 8360 /// Custom lowers integer to floating point conversions to use 8361 /// the direct move instructions available in ISA 2.07 to avoid the 8362 /// need for load/store combinations. 8363 SDValue PPCTargetLowering::LowerINT_TO_FPDirectMove(SDValue Op, 8364 SelectionDAG &DAG, 8365 const SDLoc &dl) const { 8366 assert((Op.getValueType() == MVT::f32 || 8367 Op.getValueType() == MVT::f64) && 8368 "Invalid floating point type as target of conversion"); 8369 assert(Subtarget.hasFPCVT() && 8370 "Int to FP conversions with direct moves require FPCVT"); 8371 SDValue Src = Op.getOperand(Op->isStrictFPOpcode() ? 1 : 0); 8372 bool WordInt = Src.getSimpleValueType().SimpleTy == MVT::i32; 8373 bool Signed = Op.getOpcode() == ISD::SINT_TO_FP || 8374 Op.getOpcode() == ISD::STRICT_SINT_TO_FP; 8375 unsigned MovOpc = (WordInt && !Signed) ? PPCISD::MTVSRZ : PPCISD::MTVSRA; 8376 SDValue Mov = DAG.getNode(MovOpc, dl, MVT::f64, Src); 8377 return convertIntToFP(Op, Mov, DAG, Subtarget); 8378 } 8379 8380 static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl) { 8381 8382 EVT VecVT = Vec.getValueType(); 8383 assert(VecVT.isVector() && "Expected a vector type."); 8384 assert(VecVT.getSizeInBits() < 128 && "Vector is already full width."); 8385 8386 EVT EltVT = VecVT.getVectorElementType(); 8387 unsigned WideNumElts = 128 / EltVT.getSizeInBits(); 8388 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT, WideNumElts); 8389 8390 unsigned NumConcat = WideNumElts / VecVT.getVectorNumElements(); 8391 SmallVector<SDValue, 16> Ops(NumConcat); 8392 Ops[0] = Vec; 8393 SDValue UndefVec = DAG.getUNDEF(VecVT); 8394 for (unsigned i = 1; i < NumConcat; ++i) 8395 Ops[i] = UndefVec; 8396 8397 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, Ops); 8398 } 8399 8400 SDValue PPCTargetLowering::LowerINT_TO_FPVector(SDValue Op, SelectionDAG &DAG, 8401 const SDLoc &dl) const { 8402 bool IsStrict = Op->isStrictFPOpcode(); 8403 unsigned Opc = Op.getOpcode(); 8404 SDValue Src = Op.getOperand(IsStrict ? 1 : 0); 8405 assert((Opc == ISD::UINT_TO_FP || Opc == ISD::SINT_TO_FP || 8406 Opc == ISD::STRICT_UINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP) && 8407 "Unexpected conversion type"); 8408 assert((Op.getValueType() == MVT::v2f64 || Op.getValueType() == MVT::v4f32) && 8409 "Supports conversions to v2f64/v4f32 only."); 8410 8411 // TODO: Any other flags to propagate? 8412 SDNodeFlags Flags; 8413 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept()); 8414 8415 bool SignedConv = Opc == ISD::SINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP; 8416 bool FourEltRes = Op.getValueType() == MVT::v4f32; 8417 8418 SDValue Wide = widenVec(DAG, Src, dl); 8419 EVT WideVT = Wide.getValueType(); 8420 unsigned WideNumElts = WideVT.getVectorNumElements(); 8421 MVT IntermediateVT = FourEltRes ? MVT::v4i32 : MVT::v2i64; 8422 8423 SmallVector<int, 16> ShuffV; 8424 for (unsigned i = 0; i < WideNumElts; ++i) 8425 ShuffV.push_back(i + WideNumElts); 8426 8427 int Stride = FourEltRes ? WideNumElts / 4 : WideNumElts / 2; 8428 int SaveElts = FourEltRes ? 4 : 2; 8429 if (Subtarget.isLittleEndian()) 8430 for (int i = 0; i < SaveElts; i++) 8431 ShuffV[i * Stride] = i; 8432 else 8433 for (int i = 1; i <= SaveElts; i++) 8434 ShuffV[i * Stride - 1] = i - 1; 8435 8436 SDValue ShuffleSrc2 = 8437 SignedConv ? DAG.getUNDEF(WideVT) : DAG.getConstant(0, dl, WideVT); 8438 SDValue Arrange = DAG.getVectorShuffle(WideVT, dl, Wide, ShuffleSrc2, ShuffV); 8439 8440 SDValue Extend; 8441 if (SignedConv) { 8442 Arrange = DAG.getBitcast(IntermediateVT, Arrange); 8443 EVT ExtVT = Src.getValueType(); 8444 if (Subtarget.hasP9Altivec()) 8445 ExtVT = EVT::getVectorVT(*DAG.getContext(), WideVT.getVectorElementType(), 8446 IntermediateVT.getVectorNumElements()); 8447 8448 Extend = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, IntermediateVT, Arrange, 8449 DAG.getValueType(ExtVT)); 8450 } else 8451 Extend = DAG.getNode(ISD::BITCAST, dl, IntermediateVT, Arrange); 8452 8453 if (IsStrict) 8454 return DAG.getNode(Opc, dl, DAG.getVTList(Op.getValueType(), MVT::Other), 8455 {Op.getOperand(0), Extend}, Flags); 8456 8457 return DAG.getNode(Opc, dl, Op.getValueType(), Extend); 8458 } 8459 8460 SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op, 8461 SelectionDAG &DAG) const { 8462 SDLoc dl(Op); 8463 bool IsSigned = Op.getOpcode() == ISD::SINT_TO_FP || 8464 Op.getOpcode() == ISD::STRICT_SINT_TO_FP; 8465 bool IsStrict = Op->isStrictFPOpcode(); 8466 SDValue Src = Op.getOperand(IsStrict ? 1 : 0); 8467 SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode(); 8468 8469 // TODO: Any other flags to propagate? 8470 SDNodeFlags Flags; 8471 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept()); 8472 8473 EVT InVT = Src.getValueType(); 8474 EVT OutVT = Op.getValueType(); 8475 if (OutVT.isVector() && OutVT.isFloatingPoint() && 8476 isOperationCustom(Op.getOpcode(), InVT)) 8477 return LowerINT_TO_FPVector(Op, DAG, dl); 8478 8479 // Conversions to f128 are legal. 8480 if (Op.getValueType() == MVT::f128) 8481 return Subtarget.hasP9Vector() ? Op : SDValue(); 8482 8483 // Don't handle ppc_fp128 here; let it be lowered to a libcall. 8484 if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64) 8485 return SDValue(); 8486 8487 if (Src.getValueType() == MVT::i1) { 8488 SDValue Sel = DAG.getNode(ISD::SELECT, dl, Op.getValueType(), Src, 8489 DAG.getConstantFP(1.0, dl, Op.getValueType()), 8490 DAG.getConstantFP(0.0, dl, Op.getValueType())); 8491 if (IsStrict) 8492 return DAG.getMergeValues({Sel, Chain}, dl); 8493 else 8494 return Sel; 8495 } 8496 8497 // If we have direct moves, we can do all the conversion, skip the store/load 8498 // however, without FPCVT we can't do most conversions. 8499 if (Subtarget.hasDirectMove() && directMoveIsProfitable(Op) && 8500 Subtarget.isPPC64() && Subtarget.hasFPCVT()) 8501 return LowerINT_TO_FPDirectMove(Op, DAG, dl); 8502 8503 assert((IsSigned || Subtarget.hasFPCVT()) && 8504 "UINT_TO_FP is supported only with FPCVT"); 8505 8506 if (Src.getValueType() == MVT::i64) { 8507 SDValue SINT = Src; 8508 // When converting to single-precision, we actually need to convert 8509 // to double-precision first and then round to single-precision. 8510 // To avoid double-rounding effects during that operation, we have 8511 // to prepare the input operand. Bits that might be truncated when 8512 // converting to double-precision are replaced by a bit that won't 8513 // be lost at this stage, but is below the single-precision rounding 8514 // position. 8515 // 8516 // However, if -enable-unsafe-fp-math is in effect, accept double 8517 // rounding to avoid the extra overhead. 8518 if (Op.getValueType() == MVT::f32 && 8519 !Subtarget.hasFPCVT() && 8520 !DAG.getTarget().Options.UnsafeFPMath) { 8521 8522 // Twiddle input to make sure the low 11 bits are zero. (If this 8523 // is the case, we are guaranteed the value will fit into the 53 bit 8524 // mantissa of an IEEE double-precision value without rounding.) 8525 // If any of those low 11 bits were not zero originally, make sure 8526 // bit 12 (value 2048) is set instead, so that the final rounding 8527 // to single-precision gets the correct result. 8528 SDValue Round = DAG.getNode(ISD::AND, dl, MVT::i64, 8529 SINT, DAG.getConstant(2047, dl, MVT::i64)); 8530 Round = DAG.getNode(ISD::ADD, dl, MVT::i64, 8531 Round, DAG.getConstant(2047, dl, MVT::i64)); 8532 Round = DAG.getNode(ISD::OR, dl, MVT::i64, Round, SINT); 8533 Round = DAG.getNode(ISD::AND, dl, MVT::i64, 8534 Round, DAG.getConstant(-2048, dl, MVT::i64)); 8535 8536 // However, we cannot use that value unconditionally: if the magnitude 8537 // of the input value is small, the bit-twiddling we did above might 8538 // end up visibly changing the output. Fortunately, in that case, we 8539 // don't need to twiddle bits since the original input will convert 8540 // exactly to double-precision floating-point already. Therefore, 8541 // construct a conditional to use the original value if the top 11 8542 // bits are all sign-bit copies, and use the rounded value computed 8543 // above otherwise. 8544 SDValue Cond = DAG.getNode(ISD::SRA, dl, MVT::i64, 8545 SINT, DAG.getConstant(53, dl, MVT::i32)); 8546 Cond = DAG.getNode(ISD::ADD, dl, MVT::i64, 8547 Cond, DAG.getConstant(1, dl, MVT::i64)); 8548 Cond = DAG.getSetCC( 8549 dl, 8550 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i64), 8551 Cond, DAG.getConstant(1, dl, MVT::i64), ISD::SETUGT); 8552 8553 SINT = DAG.getNode(ISD::SELECT, dl, MVT::i64, Cond, Round, SINT); 8554 } 8555 8556 ReuseLoadInfo RLI; 8557 SDValue Bits; 8558 8559 MachineFunction &MF = DAG.getMachineFunction(); 8560 if (canReuseLoadAddress(SINT, MVT::i64, RLI, DAG)) { 8561 Bits = DAG.getLoad(MVT::f64, dl, RLI.Chain, RLI.Ptr, RLI.MPI, 8562 RLI.Alignment, RLI.MMOFlags(), RLI.AAInfo, RLI.Ranges); 8563 spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG); 8564 } else if (Subtarget.hasLFIWAX() && 8565 canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::SEXTLOAD)) { 8566 MachineMemOperand *MMO = 8567 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, 8568 RLI.Alignment, RLI.AAInfo, RLI.Ranges); 8569 SDValue Ops[] = { RLI.Chain, RLI.Ptr }; 8570 Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWAX, dl, 8571 DAG.getVTList(MVT::f64, MVT::Other), 8572 Ops, MVT::i32, MMO); 8573 spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG); 8574 } else if (Subtarget.hasFPCVT() && 8575 canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::ZEXTLOAD)) { 8576 MachineMemOperand *MMO = 8577 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, 8578 RLI.Alignment, RLI.AAInfo, RLI.Ranges); 8579 SDValue Ops[] = { RLI.Chain, RLI.Ptr }; 8580 Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWZX, dl, 8581 DAG.getVTList(MVT::f64, MVT::Other), 8582 Ops, MVT::i32, MMO); 8583 spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG); 8584 } else if (((Subtarget.hasLFIWAX() && 8585 SINT.getOpcode() == ISD::SIGN_EXTEND) || 8586 (Subtarget.hasFPCVT() && 8587 SINT.getOpcode() == ISD::ZERO_EXTEND)) && 8588 SINT.getOperand(0).getValueType() == MVT::i32) { 8589 MachineFrameInfo &MFI = MF.getFrameInfo(); 8590 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 8591 8592 int FrameIdx = MFI.CreateStackObject(4, Align(4), false); 8593 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 8594 8595 SDValue Store = DAG.getStore(Chain, dl, SINT.getOperand(0), FIdx, 8596 MachinePointerInfo::getFixedStack( 8597 DAG.getMachineFunction(), FrameIdx)); 8598 Chain = Store; 8599 8600 assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 && 8601 "Expected an i32 store"); 8602 8603 RLI.Ptr = FIdx; 8604 RLI.Chain = Chain; 8605 RLI.MPI = 8606 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 8607 RLI.Alignment = Align(4); 8608 8609 MachineMemOperand *MMO = 8610 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, 8611 RLI.Alignment, RLI.AAInfo, RLI.Ranges); 8612 SDValue Ops[] = { RLI.Chain, RLI.Ptr }; 8613 Bits = DAG.getMemIntrinsicNode(SINT.getOpcode() == ISD::ZERO_EXTEND ? 8614 PPCISD::LFIWZX : PPCISD::LFIWAX, 8615 dl, DAG.getVTList(MVT::f64, MVT::Other), 8616 Ops, MVT::i32, MMO); 8617 Chain = Bits.getValue(1); 8618 } else 8619 Bits = DAG.getNode(ISD::BITCAST, dl, MVT::f64, SINT); 8620 8621 SDValue FP = convertIntToFP(Op, Bits, DAG, Subtarget, Chain); 8622 if (IsStrict) 8623 Chain = FP.getValue(1); 8624 8625 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) { 8626 if (IsStrict) 8627 FP = DAG.getNode(ISD::STRICT_FP_ROUND, dl, 8628 DAG.getVTList(MVT::f32, MVT::Other), 8629 {Chain, FP, DAG.getIntPtrConstant(0, dl)}, Flags); 8630 else 8631 FP = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, FP, 8632 DAG.getIntPtrConstant(0, dl)); 8633 } 8634 return FP; 8635 } 8636 8637 assert(Src.getValueType() == MVT::i32 && 8638 "Unhandled INT_TO_FP type in custom expander!"); 8639 // Since we only generate this in 64-bit mode, we can take advantage of 8640 // 64-bit registers. In particular, sign extend the input value into the 8641 // 64-bit register with extsw, store the WHOLE 64-bit value into the stack 8642 // then lfd it and fcfid it. 8643 MachineFunction &MF = DAG.getMachineFunction(); 8644 MachineFrameInfo &MFI = MF.getFrameInfo(); 8645 EVT PtrVT = getPointerTy(MF.getDataLayout()); 8646 8647 SDValue Ld; 8648 if (Subtarget.hasLFIWAX() || Subtarget.hasFPCVT()) { 8649 ReuseLoadInfo RLI; 8650 bool ReusingLoad; 8651 if (!(ReusingLoad = canReuseLoadAddress(Src, MVT::i32, RLI, DAG))) { 8652 int FrameIdx = MFI.CreateStackObject(4, Align(4), false); 8653 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 8654 8655 SDValue Store = DAG.getStore(Chain, dl, Src, FIdx, 8656 MachinePointerInfo::getFixedStack( 8657 DAG.getMachineFunction(), FrameIdx)); 8658 Chain = Store; 8659 8660 assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 && 8661 "Expected an i32 store"); 8662 8663 RLI.Ptr = FIdx; 8664 RLI.Chain = Chain; 8665 RLI.MPI = 8666 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 8667 RLI.Alignment = Align(4); 8668 } 8669 8670 MachineMemOperand *MMO = 8671 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, 8672 RLI.Alignment, RLI.AAInfo, RLI.Ranges); 8673 SDValue Ops[] = { RLI.Chain, RLI.Ptr }; 8674 Ld = DAG.getMemIntrinsicNode(IsSigned ? PPCISD::LFIWAX : PPCISD::LFIWZX, dl, 8675 DAG.getVTList(MVT::f64, MVT::Other), Ops, 8676 MVT::i32, MMO); 8677 Chain = Ld.getValue(1); 8678 if (ReusingLoad) 8679 spliceIntoChain(RLI.ResChain, Ld.getValue(1), DAG); 8680 } else { 8681 assert(Subtarget.isPPC64() && 8682 "i32->FP without LFIWAX supported only on PPC64"); 8683 8684 int FrameIdx = MFI.CreateStackObject(8, Align(8), false); 8685 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 8686 8687 SDValue Ext64 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i64, Src); 8688 8689 // STD the extended value into the stack slot. 8690 SDValue Store = DAG.getStore( 8691 Chain, dl, Ext64, FIdx, 8692 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx)); 8693 Chain = Store; 8694 8695 // Load the value as a double. 8696 Ld = DAG.getLoad( 8697 MVT::f64, dl, Chain, FIdx, 8698 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx)); 8699 Chain = Ld.getValue(1); 8700 } 8701 8702 // FCFID it and return it. 8703 SDValue FP = convertIntToFP(Op, Ld, DAG, Subtarget, Chain); 8704 if (IsStrict) 8705 Chain = FP.getValue(1); 8706 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) { 8707 if (IsStrict) 8708 FP = DAG.getNode(ISD::STRICT_FP_ROUND, dl, 8709 DAG.getVTList(MVT::f32, MVT::Other), 8710 {Chain, FP, DAG.getIntPtrConstant(0, dl)}, Flags); 8711 else 8712 FP = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, FP, 8713 DAG.getIntPtrConstant(0, dl)); 8714 } 8715 return FP; 8716 } 8717 8718 SDValue PPCTargetLowering::LowerFLT_ROUNDS_(SDValue Op, 8719 SelectionDAG &DAG) const { 8720 SDLoc dl(Op); 8721 /* 8722 The rounding mode is in bits 30:31 of FPSR, and has the following 8723 settings: 8724 00 Round to nearest 8725 01 Round to 0 8726 10 Round to +inf 8727 11 Round to -inf 8728 8729 FLT_ROUNDS, on the other hand, expects the following: 8730 -1 Undefined 8731 0 Round to 0 8732 1 Round to nearest 8733 2 Round to +inf 8734 3 Round to -inf 8735 8736 To perform the conversion, we do: 8737 ((FPSCR & 0x3) ^ ((~FPSCR & 0x3) >> 1)) 8738 */ 8739 8740 MachineFunction &MF = DAG.getMachineFunction(); 8741 EVT VT = Op.getValueType(); 8742 EVT PtrVT = getPointerTy(MF.getDataLayout()); 8743 8744 // Save FP Control Word to register 8745 SDValue Chain = Op.getOperand(0); 8746 SDValue MFFS = DAG.getNode(PPCISD::MFFS, dl, {MVT::f64, MVT::Other}, Chain); 8747 Chain = MFFS.getValue(1); 8748 8749 SDValue CWD; 8750 if (isTypeLegal(MVT::i64)) { 8751 CWD = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, 8752 DAG.getNode(ISD::BITCAST, dl, MVT::i64, MFFS)); 8753 } else { 8754 // Save FP register to stack slot 8755 int SSFI = MF.getFrameInfo().CreateStackObject(8, Align(8), false); 8756 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT); 8757 Chain = DAG.getStore(Chain, dl, MFFS, StackSlot, MachinePointerInfo()); 8758 8759 // Load FP Control Word from low 32 bits of stack slot. 8760 assert(hasBigEndianPartOrdering(MVT::i64, MF.getDataLayout()) && 8761 "Stack slot adjustment is valid only on big endian subtargets!"); 8762 SDValue Four = DAG.getConstant(4, dl, PtrVT); 8763 SDValue Addr = DAG.getNode(ISD::ADD, dl, PtrVT, StackSlot, Four); 8764 CWD = DAG.getLoad(MVT::i32, dl, Chain, Addr, MachinePointerInfo()); 8765 Chain = CWD.getValue(1); 8766 } 8767 8768 // Transform as necessary 8769 SDValue CWD1 = 8770 DAG.getNode(ISD::AND, dl, MVT::i32, 8771 CWD, DAG.getConstant(3, dl, MVT::i32)); 8772 SDValue CWD2 = 8773 DAG.getNode(ISD::SRL, dl, MVT::i32, 8774 DAG.getNode(ISD::AND, dl, MVT::i32, 8775 DAG.getNode(ISD::XOR, dl, MVT::i32, 8776 CWD, DAG.getConstant(3, dl, MVT::i32)), 8777 DAG.getConstant(3, dl, MVT::i32)), 8778 DAG.getConstant(1, dl, MVT::i32)); 8779 8780 SDValue RetVal = 8781 DAG.getNode(ISD::XOR, dl, MVT::i32, CWD1, CWD2); 8782 8783 RetVal = 8784 DAG.getNode((VT.getSizeInBits() < 16 ? ISD::TRUNCATE : ISD::ZERO_EXTEND), 8785 dl, VT, RetVal); 8786 8787 return DAG.getMergeValues({RetVal, Chain}, dl); 8788 } 8789 8790 SDValue PPCTargetLowering::LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG) const { 8791 EVT VT = Op.getValueType(); 8792 unsigned BitWidth = VT.getSizeInBits(); 8793 SDLoc dl(Op); 8794 assert(Op.getNumOperands() == 3 && 8795 VT == Op.getOperand(1).getValueType() && 8796 "Unexpected SHL!"); 8797 8798 // Expand into a bunch of logical ops. Note that these ops 8799 // depend on the PPC behavior for oversized shift amounts. 8800 SDValue Lo = Op.getOperand(0); 8801 SDValue Hi = Op.getOperand(1); 8802 SDValue Amt = Op.getOperand(2); 8803 EVT AmtVT = Amt.getValueType(); 8804 8805 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT, 8806 DAG.getConstant(BitWidth, dl, AmtVT), Amt); 8807 SDValue Tmp2 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Amt); 8808 SDValue Tmp3 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Tmp1); 8809 SDValue Tmp4 = DAG.getNode(ISD::OR , dl, VT, Tmp2, Tmp3); 8810 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt, 8811 DAG.getConstant(-BitWidth, dl, AmtVT)); 8812 SDValue Tmp6 = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Tmp5); 8813 SDValue OutHi = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6); 8814 SDValue OutLo = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Amt); 8815 SDValue OutOps[] = { OutLo, OutHi }; 8816 return DAG.getMergeValues(OutOps, dl); 8817 } 8818 8819 SDValue PPCTargetLowering::LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const { 8820 EVT VT = Op.getValueType(); 8821 SDLoc dl(Op); 8822 unsigned BitWidth = VT.getSizeInBits(); 8823 assert(Op.getNumOperands() == 3 && 8824 VT == Op.getOperand(1).getValueType() && 8825 "Unexpected SRL!"); 8826 8827 // Expand into a bunch of logical ops. Note that these ops 8828 // depend on the PPC behavior for oversized shift amounts. 8829 SDValue Lo = Op.getOperand(0); 8830 SDValue Hi = Op.getOperand(1); 8831 SDValue Amt = Op.getOperand(2); 8832 EVT AmtVT = Amt.getValueType(); 8833 8834 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT, 8835 DAG.getConstant(BitWidth, dl, AmtVT), Amt); 8836 SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt); 8837 SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1); 8838 SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); 8839 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt, 8840 DAG.getConstant(-BitWidth, dl, AmtVT)); 8841 SDValue Tmp6 = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Tmp5); 8842 SDValue OutLo = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6); 8843 SDValue OutHi = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Amt); 8844 SDValue OutOps[] = { OutLo, OutHi }; 8845 return DAG.getMergeValues(OutOps, dl); 8846 } 8847 8848 SDValue PPCTargetLowering::LowerSRA_PARTS(SDValue Op, SelectionDAG &DAG) const { 8849 SDLoc dl(Op); 8850 EVT VT = Op.getValueType(); 8851 unsigned BitWidth = VT.getSizeInBits(); 8852 assert(Op.getNumOperands() == 3 && 8853 VT == Op.getOperand(1).getValueType() && 8854 "Unexpected SRA!"); 8855 8856 // Expand into a bunch of logical ops, followed by a select_cc. 8857 SDValue Lo = Op.getOperand(0); 8858 SDValue Hi = Op.getOperand(1); 8859 SDValue Amt = Op.getOperand(2); 8860 EVT AmtVT = Amt.getValueType(); 8861 8862 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT, 8863 DAG.getConstant(BitWidth, dl, AmtVT), Amt); 8864 SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt); 8865 SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1); 8866 SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); 8867 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt, 8868 DAG.getConstant(-BitWidth, dl, AmtVT)); 8869 SDValue Tmp6 = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Tmp5); 8870 SDValue OutHi = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Amt); 8871 SDValue OutLo = DAG.getSelectCC(dl, Tmp5, DAG.getConstant(0, dl, AmtVT), 8872 Tmp4, Tmp6, ISD::SETLE); 8873 SDValue OutOps[] = { OutLo, OutHi }; 8874 return DAG.getMergeValues(OutOps, dl); 8875 } 8876 8877 SDValue PPCTargetLowering::LowerFunnelShift(SDValue Op, 8878 SelectionDAG &DAG) const { 8879 SDLoc dl(Op); 8880 EVT VT = Op.getValueType(); 8881 unsigned BitWidth = VT.getSizeInBits(); 8882 8883 bool IsFSHL = Op.getOpcode() == ISD::FSHL; 8884 SDValue X = Op.getOperand(0); 8885 SDValue Y = Op.getOperand(1); 8886 SDValue Z = Op.getOperand(2); 8887 EVT AmtVT = Z.getValueType(); 8888 8889 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW))) 8890 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW)) 8891 // This is simpler than TargetLowering::expandFunnelShift because we can rely 8892 // on PowerPC shift by BW being well defined. 8893 Z = DAG.getNode(ISD::AND, dl, AmtVT, Z, 8894 DAG.getConstant(BitWidth - 1, dl, AmtVT)); 8895 SDValue SubZ = 8896 DAG.getNode(ISD::SUB, dl, AmtVT, DAG.getConstant(BitWidth, dl, AmtVT), Z); 8897 X = DAG.getNode(PPCISD::SHL, dl, VT, X, IsFSHL ? Z : SubZ); 8898 Y = DAG.getNode(PPCISD::SRL, dl, VT, Y, IsFSHL ? SubZ : Z); 8899 return DAG.getNode(ISD::OR, dl, VT, X, Y); 8900 } 8901 8902 //===----------------------------------------------------------------------===// 8903 // Vector related lowering. 8904 // 8905 8906 /// getCanonicalConstSplat - Build a canonical splat immediate of Val with an 8907 /// element size of SplatSize. Cast the result to VT. 8908 static SDValue getCanonicalConstSplat(uint64_t Val, unsigned SplatSize, EVT VT, 8909 SelectionDAG &DAG, const SDLoc &dl) { 8910 static const MVT VTys[] = { // canonical VT to use for each size. 8911 MVT::v16i8, MVT::v8i16, MVT::Other, MVT::v4i32 8912 }; 8913 8914 EVT ReqVT = VT != MVT::Other ? VT : VTys[SplatSize-1]; 8915 8916 // For a splat with all ones, turn it to vspltisb 0xFF to canonicalize. 8917 if (Val == ((1LLU << (SplatSize * 8)) - 1)) { 8918 SplatSize = 1; 8919 Val = 0xFF; 8920 } 8921 8922 EVT CanonicalVT = VTys[SplatSize-1]; 8923 8924 // Build a canonical splat for this value. 8925 return DAG.getBitcast(ReqVT, DAG.getConstant(Val, dl, CanonicalVT)); 8926 } 8927 8928 /// BuildIntrinsicOp - Return a unary operator intrinsic node with the 8929 /// specified intrinsic ID. 8930 static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op, SelectionDAG &DAG, 8931 const SDLoc &dl, EVT DestVT = MVT::Other) { 8932 if (DestVT == MVT::Other) DestVT = Op.getValueType(); 8933 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT, 8934 DAG.getConstant(IID, dl, MVT::i32), Op); 8935 } 8936 8937 /// BuildIntrinsicOp - Return a binary operator intrinsic node with the 8938 /// specified intrinsic ID. 8939 static SDValue BuildIntrinsicOp(unsigned IID, SDValue LHS, SDValue RHS, 8940 SelectionDAG &DAG, const SDLoc &dl, 8941 EVT DestVT = MVT::Other) { 8942 if (DestVT == MVT::Other) DestVT = LHS.getValueType(); 8943 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT, 8944 DAG.getConstant(IID, dl, MVT::i32), LHS, RHS); 8945 } 8946 8947 /// BuildIntrinsicOp - Return a ternary operator intrinsic node with the 8948 /// specified intrinsic ID. 8949 static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op0, SDValue Op1, 8950 SDValue Op2, SelectionDAG &DAG, const SDLoc &dl, 8951 EVT DestVT = MVT::Other) { 8952 if (DestVT == MVT::Other) DestVT = Op0.getValueType(); 8953 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT, 8954 DAG.getConstant(IID, dl, MVT::i32), Op0, Op1, Op2); 8955 } 8956 8957 /// BuildVSLDOI - Return a VECTOR_SHUFFLE that is a vsldoi of the specified 8958 /// amount. The result has the specified value type. 8959 static SDValue BuildVSLDOI(SDValue LHS, SDValue RHS, unsigned Amt, EVT VT, 8960 SelectionDAG &DAG, const SDLoc &dl) { 8961 // Force LHS/RHS to be the right type. 8962 LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, LHS); 8963 RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, RHS); 8964 8965 int Ops[16]; 8966 for (unsigned i = 0; i != 16; ++i) 8967 Ops[i] = i + Amt; 8968 SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, LHS, RHS, Ops); 8969 return DAG.getNode(ISD::BITCAST, dl, VT, T); 8970 } 8971 8972 /// Do we have an efficient pattern in a .td file for this node? 8973 /// 8974 /// \param V - pointer to the BuildVectorSDNode being matched 8975 /// \param HasDirectMove - does this subtarget have VSR <-> GPR direct moves? 8976 /// 8977 /// There are some patterns where it is beneficial to keep a BUILD_VECTOR 8978 /// node as a BUILD_VECTOR node rather than expanding it. The patterns where 8979 /// the opposite is true (expansion is beneficial) are: 8980 /// - The node builds a vector out of integers that are not 32 or 64-bits 8981 /// - The node builds a vector out of constants 8982 /// - The node is a "load-and-splat" 8983 /// In all other cases, we will choose to keep the BUILD_VECTOR. 8984 static bool haveEfficientBuildVectorPattern(BuildVectorSDNode *V, 8985 bool HasDirectMove, 8986 bool HasP8Vector) { 8987 EVT VecVT = V->getValueType(0); 8988 bool RightType = VecVT == MVT::v2f64 || 8989 (HasP8Vector && VecVT == MVT::v4f32) || 8990 (HasDirectMove && (VecVT == MVT::v2i64 || VecVT == MVT::v4i32)); 8991 if (!RightType) 8992 return false; 8993 8994 bool IsSplat = true; 8995 bool IsLoad = false; 8996 SDValue Op0 = V->getOperand(0); 8997 8998 // This function is called in a block that confirms the node is not a constant 8999 // splat. So a constant BUILD_VECTOR here means the vector is built out of 9000 // different constants. 9001 if (V->isConstant()) 9002 return false; 9003 for (int i = 0, e = V->getNumOperands(); i < e; ++i) { 9004 if (V->getOperand(i).isUndef()) 9005 return false; 9006 // We want to expand nodes that represent load-and-splat even if the 9007 // loaded value is a floating point truncation or conversion to int. 9008 if (V->getOperand(i).getOpcode() == ISD::LOAD || 9009 (V->getOperand(i).getOpcode() == ISD::FP_ROUND && 9010 V->getOperand(i).getOperand(0).getOpcode() == ISD::LOAD) || 9011 (V->getOperand(i).getOpcode() == ISD::FP_TO_SINT && 9012 V->getOperand(i).getOperand(0).getOpcode() == ISD::LOAD) || 9013 (V->getOperand(i).getOpcode() == ISD::FP_TO_UINT && 9014 V->getOperand(i).getOperand(0).getOpcode() == ISD::LOAD)) 9015 IsLoad = true; 9016 // If the operands are different or the input is not a load and has more 9017 // uses than just this BV node, then it isn't a splat. 9018 if (V->getOperand(i) != Op0 || 9019 (!IsLoad && !V->isOnlyUserOf(V->getOperand(i).getNode()))) 9020 IsSplat = false; 9021 } 9022 return !(IsSplat && IsLoad); 9023 } 9024 9025 // Lower BITCAST(f128, (build_pair i64, i64)) to BUILD_FP128. 9026 SDValue PPCTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const { 9027 9028 SDLoc dl(Op); 9029 SDValue Op0 = Op->getOperand(0); 9030 9031 if ((Op.getValueType() != MVT::f128) || 9032 (Op0.getOpcode() != ISD::BUILD_PAIR) || 9033 (Op0.getOperand(0).getValueType() != MVT::i64) || 9034 (Op0.getOperand(1).getValueType() != MVT::i64)) 9035 return SDValue(); 9036 9037 return DAG.getNode(PPCISD::BUILD_FP128, dl, MVT::f128, Op0.getOperand(0), 9038 Op0.getOperand(1)); 9039 } 9040 9041 static const SDValue *getNormalLoadInput(const SDValue &Op, bool &IsPermuted) { 9042 const SDValue *InputLoad = &Op; 9043 if (InputLoad->getOpcode() == ISD::BITCAST) 9044 InputLoad = &InputLoad->getOperand(0); 9045 if (InputLoad->getOpcode() == ISD::SCALAR_TO_VECTOR || 9046 InputLoad->getOpcode() == PPCISD::SCALAR_TO_VECTOR_PERMUTED) { 9047 IsPermuted = InputLoad->getOpcode() == PPCISD::SCALAR_TO_VECTOR_PERMUTED; 9048 InputLoad = &InputLoad->getOperand(0); 9049 } 9050 if (InputLoad->getOpcode() != ISD::LOAD) 9051 return nullptr; 9052 LoadSDNode *LD = cast<LoadSDNode>(*InputLoad); 9053 return ISD::isNormalLoad(LD) ? InputLoad : nullptr; 9054 } 9055 9056 // Convert the argument APFloat to a single precision APFloat if there is no 9057 // loss in information during the conversion to single precision APFloat and the 9058 // resulting number is not a denormal number. Return true if successful. 9059 bool llvm::convertToNonDenormSingle(APFloat &ArgAPFloat) { 9060 APFloat APFloatToConvert = ArgAPFloat; 9061 bool LosesInfo = true; 9062 APFloatToConvert.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven, 9063 &LosesInfo); 9064 bool Success = (!LosesInfo && !APFloatToConvert.isDenormal()); 9065 if (Success) 9066 ArgAPFloat = APFloatToConvert; 9067 return Success; 9068 } 9069 9070 // Bitcast the argument APInt to a double and convert it to a single precision 9071 // APFloat, bitcast the APFloat to an APInt and assign it to the original 9072 // argument if there is no loss in information during the conversion from 9073 // double to single precision APFloat and the resulting number is not a denormal 9074 // number. Return true if successful. 9075 bool llvm::convertToNonDenormSingle(APInt &ArgAPInt) { 9076 double DpValue = ArgAPInt.bitsToDouble(); 9077 APFloat APFloatDp(DpValue); 9078 bool Success = convertToNonDenormSingle(APFloatDp); 9079 if (Success) 9080 ArgAPInt = APFloatDp.bitcastToAPInt(); 9081 return Success; 9082 } 9083 9084 // Nondestructive check for convertTonNonDenormSingle. 9085 bool llvm::checkConvertToNonDenormSingle(APFloat &ArgAPFloat) { 9086 // Only convert if it loses info, since XXSPLTIDP should 9087 // handle the other case. 9088 APFloat APFloatToConvert = ArgAPFloat; 9089 bool LosesInfo = true; 9090 APFloatToConvert.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven, 9091 &LosesInfo); 9092 9093 return (!LosesInfo && !APFloatToConvert.isDenormal()); 9094 } 9095 9096 static bool isValidSplatLoad(const PPCSubtarget &Subtarget, const SDValue &Op, 9097 unsigned &Opcode) { 9098 LoadSDNode *InputNode = dyn_cast<LoadSDNode>(Op.getOperand(0)); 9099 if (!InputNode || !Subtarget.hasVSX() || !ISD::isUNINDEXEDLoad(InputNode)) 9100 return false; 9101 9102 EVT Ty = Op->getValueType(0); 9103 // For v2f64, v4f32 and v4i32 types, we require the load to be non-extending 9104 // as we cannot handle extending loads for these types. 9105 if ((Ty == MVT::v2f64 || Ty == MVT::v4f32 || Ty == MVT::v4i32) && 9106 ISD::isNON_EXTLoad(InputNode)) 9107 return true; 9108 9109 EVT MemVT = InputNode->getMemoryVT(); 9110 // For v8i16 and v16i8 types, extending loads can be handled as long as the 9111 // memory VT is the same vector element VT type. 9112 // The loads feeding into the v8i16 and v16i8 types will be extending because 9113 // scalar i8/i16 are not legal types. 9114 if ((Ty == MVT::v8i16 || Ty == MVT::v16i8) && ISD::isEXTLoad(InputNode) && 9115 (MemVT == Ty.getVectorElementType())) 9116 return true; 9117 9118 if (Ty == MVT::v2i64) { 9119 // Check the extend type, when the input type is i32, and the output vector 9120 // type is v2i64. 9121 if (MemVT == MVT::i32) { 9122 if (ISD::isZEXTLoad(InputNode)) 9123 Opcode = PPCISD::ZEXT_LD_SPLAT; 9124 if (ISD::isSEXTLoad(InputNode)) 9125 Opcode = PPCISD::SEXT_LD_SPLAT; 9126 } 9127 return true; 9128 } 9129 return false; 9130 } 9131 9132 // If this is a case we can't handle, return null and let the default 9133 // expansion code take care of it. If we CAN select this case, and if it 9134 // selects to a single instruction, return Op. Otherwise, if we can codegen 9135 // this case more efficiently than a constant pool load, lower it to the 9136 // sequence of ops that should be used. 9137 SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op, 9138 SelectionDAG &DAG) const { 9139 SDLoc dl(Op); 9140 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode()); 9141 assert(BVN && "Expected a BuildVectorSDNode in LowerBUILD_VECTOR"); 9142 9143 // Check if this is a splat of a constant value. 9144 APInt APSplatBits, APSplatUndef; 9145 unsigned SplatBitSize; 9146 bool HasAnyUndefs; 9147 bool BVNIsConstantSplat = 9148 BVN->isConstantSplat(APSplatBits, APSplatUndef, SplatBitSize, 9149 HasAnyUndefs, 0, !Subtarget.isLittleEndian()); 9150 9151 // If it is a splat of a double, check if we can shrink it to a 32 bit 9152 // non-denormal float which when converted back to double gives us the same 9153 // double. This is to exploit the XXSPLTIDP instruction. 9154 // If we lose precision, we use XXSPLTI32DX. 9155 if (BVNIsConstantSplat && (SplatBitSize == 64) && 9156 Subtarget.hasPrefixInstrs()) { 9157 // Check the type first to short-circuit so we don't modify APSplatBits if 9158 // this block isn't executed. 9159 if ((Op->getValueType(0) == MVT::v2f64) && 9160 convertToNonDenormSingle(APSplatBits)) { 9161 SDValue SplatNode = DAG.getNode( 9162 PPCISD::XXSPLTI_SP_TO_DP, dl, MVT::v2f64, 9163 DAG.getTargetConstant(APSplatBits.getZExtValue(), dl, MVT::i32)); 9164 return DAG.getBitcast(Op.getValueType(), SplatNode); 9165 } else { 9166 // We may lose precision, so we have to use XXSPLTI32DX. 9167 9168 uint32_t Hi = 9169 (uint32_t)((APSplatBits.getZExtValue() & 0xFFFFFFFF00000000LL) >> 32); 9170 uint32_t Lo = 9171 (uint32_t)(APSplatBits.getZExtValue() & 0xFFFFFFFF); 9172 SDValue SplatNode = DAG.getUNDEF(MVT::v2i64); 9173 9174 if (!Hi || !Lo) 9175 // If either load is 0, then we should generate XXLXOR to set to 0. 9176 SplatNode = DAG.getTargetConstant(0, dl, MVT::v2i64); 9177 9178 if (Hi) 9179 SplatNode = DAG.getNode( 9180 PPCISD::XXSPLTI32DX, dl, MVT::v2i64, SplatNode, 9181 DAG.getTargetConstant(0, dl, MVT::i32), 9182 DAG.getTargetConstant(Hi, dl, MVT::i32)); 9183 9184 if (Lo) 9185 SplatNode = 9186 DAG.getNode(PPCISD::XXSPLTI32DX, dl, MVT::v2i64, SplatNode, 9187 DAG.getTargetConstant(1, dl, MVT::i32), 9188 DAG.getTargetConstant(Lo, dl, MVT::i32)); 9189 9190 return DAG.getBitcast(Op.getValueType(), SplatNode); 9191 } 9192 } 9193 9194 if (!BVNIsConstantSplat || SplatBitSize > 32) { 9195 unsigned NewOpcode = PPCISD::LD_SPLAT; 9196 9197 // Handle load-and-splat patterns as we have instructions that will do this 9198 // in one go. 9199 if (DAG.isSplatValue(Op, true) && 9200 isValidSplatLoad(Subtarget, Op, NewOpcode)) { 9201 const SDValue *InputLoad = &Op.getOperand(0); 9202 LoadSDNode *LD = cast<LoadSDNode>(*InputLoad); 9203 9204 // If the input load is an extending load, it will be an i32 -> i64 9205 // extending load and isValidSplatLoad() will update NewOpcode. 9206 unsigned MemorySize = LD->getMemoryVT().getScalarSizeInBits(); 9207 unsigned ElementSize = 9208 MemorySize * ((NewOpcode == PPCISD::LD_SPLAT) ? 1 : 2); 9209 9210 assert(((ElementSize == 2 * MemorySize) 9211 ? (NewOpcode == PPCISD::ZEXT_LD_SPLAT || 9212 NewOpcode == PPCISD::SEXT_LD_SPLAT) 9213 : (NewOpcode == PPCISD::LD_SPLAT)) && 9214 "Unmatched element size and opcode!\n"); 9215 9216 // Checking for a single use of this load, we have to check for vector 9217 // width (128 bits) / ElementSize uses (since each operand of the 9218 // BUILD_VECTOR is a separate use of the value. 9219 unsigned NumUsesOfInputLD = 128 / ElementSize; 9220 for (SDValue BVInOp : Op->ops()) 9221 if (BVInOp.isUndef()) 9222 NumUsesOfInputLD--; 9223 9224 // Exclude somes case where LD_SPLAT is worse than scalar_to_vector: 9225 // Below cases should also happen for "lfiwzx/lfiwax + LE target + index 9226 // 1" and "lxvrhx + BE target + index 7" and "lxvrbx + BE target + index 9227 // 15", but funciton IsValidSplatLoad() now will only return true when 9228 // the data at index 0 is not nullptr. So we will not get into trouble for 9229 // these cases. 9230 // 9231 // case 1 - lfiwzx/lfiwax 9232 // 1.1: load result is i32 and is sign/zero extend to i64; 9233 // 1.2: build a v2i64 vector type with above loaded value; 9234 // 1.3: the vector has only one value at index 0, others are all undef; 9235 // 1.4: on BE target, so that lfiwzx/lfiwax does not need any permute. 9236 if (NumUsesOfInputLD == 1 && 9237 (Op->getValueType(0) == MVT::v2i64 && NewOpcode != PPCISD::LD_SPLAT && 9238 !Subtarget.isLittleEndian() && Subtarget.hasVSX() && 9239 Subtarget.hasLFIWAX())) 9240 return SDValue(); 9241 9242 // case 2 - lxvr[hb]x 9243 // 2.1: load result is at most i16; 9244 // 2.2: build a vector with above loaded value; 9245 // 2.3: the vector has only one value at index 0, others are all undef; 9246 // 2.4: on LE target, so that lxvr[hb]x does not need any permute. 9247 if (NumUsesOfInputLD == 1 && Subtarget.isLittleEndian() && 9248 Subtarget.isISA3_1() && ElementSize <= 16) 9249 return SDValue(); 9250 9251 assert(NumUsesOfInputLD > 0 && "No uses of input LD of a build_vector?"); 9252 if (InputLoad->getNode()->hasNUsesOfValue(NumUsesOfInputLD, 0) && 9253 Subtarget.hasVSX()) { 9254 SDValue Ops[] = { 9255 LD->getChain(), // Chain 9256 LD->getBasePtr(), // Ptr 9257 DAG.getValueType(Op.getValueType()) // VT 9258 }; 9259 SDValue LdSplt = DAG.getMemIntrinsicNode( 9260 NewOpcode, dl, DAG.getVTList(Op.getValueType(), MVT::Other), Ops, 9261 LD->getMemoryVT(), LD->getMemOperand()); 9262 // Replace all uses of the output chain of the original load with the 9263 // output chain of the new load. 9264 DAG.ReplaceAllUsesOfValueWith(InputLoad->getValue(1), 9265 LdSplt.getValue(1)); 9266 return LdSplt; 9267 } 9268 } 9269 9270 // In 64BIT mode BUILD_VECTOR nodes that are not constant splats of up to 9271 // 32-bits can be lowered to VSX instructions under certain conditions. 9272 // Without VSX, there is no pattern more efficient than expanding the node. 9273 if (Subtarget.hasVSX() && Subtarget.isPPC64() && 9274 haveEfficientBuildVectorPattern(BVN, Subtarget.hasDirectMove(), 9275 Subtarget.hasP8Vector())) 9276 return Op; 9277 return SDValue(); 9278 } 9279 9280 uint64_t SplatBits = APSplatBits.getZExtValue(); 9281 uint64_t SplatUndef = APSplatUndef.getZExtValue(); 9282 unsigned SplatSize = SplatBitSize / 8; 9283 9284 // First, handle single instruction cases. 9285 9286 // All zeros? 9287 if (SplatBits == 0) { 9288 // Canonicalize all zero vectors to be v4i32. 9289 if (Op.getValueType() != MVT::v4i32 || HasAnyUndefs) { 9290 SDValue Z = DAG.getConstant(0, dl, MVT::v4i32); 9291 Op = DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Z); 9292 } 9293 return Op; 9294 } 9295 9296 // We have XXSPLTIW for constant splats four bytes wide. 9297 // Given vector length is a multiple of 4, 2-byte splats can be replaced 9298 // with 4-byte splats. We replicate the SplatBits in case of 2-byte splat to 9299 // make a 4-byte splat element. For example: 2-byte splat of 0xABAB can be 9300 // turned into a 4-byte splat of 0xABABABAB. 9301 if (Subtarget.hasPrefixInstrs() && SplatSize == 2) 9302 return getCanonicalConstSplat(SplatBits | (SplatBits << 16), SplatSize * 2, 9303 Op.getValueType(), DAG, dl); 9304 9305 if (Subtarget.hasPrefixInstrs() && SplatSize == 4) 9306 return getCanonicalConstSplat(SplatBits, SplatSize, Op.getValueType(), DAG, 9307 dl); 9308 9309 // We have XXSPLTIB for constant splats one byte wide. 9310 if (Subtarget.hasP9Vector() && SplatSize == 1) 9311 return getCanonicalConstSplat(SplatBits, SplatSize, Op.getValueType(), DAG, 9312 dl); 9313 9314 // If the sign extended value is in the range [-16,15], use VSPLTI[bhw]. 9315 int32_t SextVal= (int32_t(SplatBits << (32-SplatBitSize)) >> 9316 (32-SplatBitSize)); 9317 if (SextVal >= -16 && SextVal <= 15) 9318 return getCanonicalConstSplat(SextVal, SplatSize, Op.getValueType(), DAG, 9319 dl); 9320 9321 // Two instruction sequences. 9322 9323 // If this value is in the range [-32,30] and is even, use: 9324 // VSPLTI[bhw](val/2) + VSPLTI[bhw](val/2) 9325 // If this value is in the range [17,31] and is odd, use: 9326 // VSPLTI[bhw](val-16) - VSPLTI[bhw](-16) 9327 // If this value is in the range [-31,-17] and is odd, use: 9328 // VSPLTI[bhw](val+16) + VSPLTI[bhw](-16) 9329 // Note the last two are three-instruction sequences. 9330 if (SextVal >= -32 && SextVal <= 31) { 9331 // To avoid having these optimizations undone by constant folding, 9332 // we convert to a pseudo that will be expanded later into one of 9333 // the above forms. 9334 SDValue Elt = DAG.getConstant(SextVal, dl, MVT::i32); 9335 EVT VT = (SplatSize == 1 ? MVT::v16i8 : 9336 (SplatSize == 2 ? MVT::v8i16 : MVT::v4i32)); 9337 SDValue EltSize = DAG.getConstant(SplatSize, dl, MVT::i32); 9338 SDValue RetVal = DAG.getNode(PPCISD::VADD_SPLAT, dl, VT, Elt, EltSize); 9339 if (VT == Op.getValueType()) 9340 return RetVal; 9341 else 9342 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), RetVal); 9343 } 9344 9345 // If this is 0x8000_0000 x 4, turn into vspltisw + vslw. If it is 9346 // 0x7FFF_FFFF x 4, turn it into not(0x8000_0000). This is important 9347 // for fneg/fabs. 9348 if (SplatSize == 4 && SplatBits == (0x7FFFFFFF&~SplatUndef)) { 9349 // Make -1 and vspltisw -1: 9350 SDValue OnesV = getCanonicalConstSplat(-1, 4, MVT::v4i32, DAG, dl); 9351 9352 // Make the VSLW intrinsic, computing 0x8000_0000. 9353 SDValue Res = BuildIntrinsicOp(Intrinsic::ppc_altivec_vslw, OnesV, 9354 OnesV, DAG, dl); 9355 9356 // xor by OnesV to invert it. 9357 Res = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Res, OnesV); 9358 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 9359 } 9360 9361 // Check to see if this is a wide variety of vsplti*, binop self cases. 9362 static const signed char SplatCsts[] = { 9363 -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7, 9364 -8, 8, -9, 9, -10, 10, -11, 11, -12, 12, -13, 13, 14, -14, 15, -15, -16 9365 }; 9366 9367 for (unsigned idx = 0; idx < array_lengthof(SplatCsts); ++idx) { 9368 // Indirect through the SplatCsts array so that we favor 'vsplti -1' for 9369 // cases which are ambiguous (e.g. formation of 0x8000_0000). 'vsplti -1' 9370 int i = SplatCsts[idx]; 9371 9372 // Figure out what shift amount will be used by altivec if shifted by i in 9373 // this splat size. 9374 unsigned TypeShiftAmt = i & (SplatBitSize-1); 9375 9376 // vsplti + shl self. 9377 if (SextVal == (int)((unsigned)i << TypeShiftAmt)) { 9378 SDValue Res = getCanonicalConstSplat(i, SplatSize, MVT::Other, DAG, dl); 9379 static const unsigned IIDs[] = { // Intrinsic to use for each size. 9380 Intrinsic::ppc_altivec_vslb, Intrinsic::ppc_altivec_vslh, 0, 9381 Intrinsic::ppc_altivec_vslw 9382 }; 9383 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); 9384 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 9385 } 9386 9387 // vsplti + srl self. 9388 if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) { 9389 SDValue Res = getCanonicalConstSplat(i, SplatSize, MVT::Other, DAG, dl); 9390 static const unsigned IIDs[] = { // Intrinsic to use for each size. 9391 Intrinsic::ppc_altivec_vsrb, Intrinsic::ppc_altivec_vsrh, 0, 9392 Intrinsic::ppc_altivec_vsrw 9393 }; 9394 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); 9395 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 9396 } 9397 9398 // vsplti + rol self. 9399 if (SextVal == (int)(((unsigned)i << TypeShiftAmt) | 9400 ((unsigned)i >> (SplatBitSize-TypeShiftAmt)))) { 9401 SDValue Res = getCanonicalConstSplat(i, SplatSize, MVT::Other, DAG, dl); 9402 static const unsigned IIDs[] = { // Intrinsic to use for each size. 9403 Intrinsic::ppc_altivec_vrlb, Intrinsic::ppc_altivec_vrlh, 0, 9404 Intrinsic::ppc_altivec_vrlw 9405 }; 9406 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); 9407 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 9408 } 9409 9410 // t = vsplti c, result = vsldoi t, t, 1 9411 if (SextVal == (int)(((unsigned)i << 8) | (i < 0 ? 0xFF : 0))) { 9412 SDValue T = getCanonicalConstSplat(i, SplatSize, MVT::v16i8, DAG, dl); 9413 unsigned Amt = Subtarget.isLittleEndian() ? 15 : 1; 9414 return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl); 9415 } 9416 // t = vsplti c, result = vsldoi t, t, 2 9417 if (SextVal == (int)(((unsigned)i << 16) | (i < 0 ? 0xFFFF : 0))) { 9418 SDValue T = getCanonicalConstSplat(i, SplatSize, MVT::v16i8, DAG, dl); 9419 unsigned Amt = Subtarget.isLittleEndian() ? 14 : 2; 9420 return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl); 9421 } 9422 // t = vsplti c, result = vsldoi t, t, 3 9423 if (SextVal == (int)(((unsigned)i << 24) | (i < 0 ? 0xFFFFFF : 0))) { 9424 SDValue T = getCanonicalConstSplat(i, SplatSize, MVT::v16i8, DAG, dl); 9425 unsigned Amt = Subtarget.isLittleEndian() ? 13 : 3; 9426 return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl); 9427 } 9428 } 9429 9430 return SDValue(); 9431 } 9432 9433 /// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit 9434 /// the specified operations to build the shuffle. 9435 static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS, 9436 SDValue RHS, SelectionDAG &DAG, 9437 const SDLoc &dl) { 9438 unsigned OpNum = (PFEntry >> 26) & 0x0F; 9439 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1); 9440 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1); 9441 9442 enum { 9443 OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3> 9444 OP_VMRGHW, 9445 OP_VMRGLW, 9446 OP_VSPLTISW0, 9447 OP_VSPLTISW1, 9448 OP_VSPLTISW2, 9449 OP_VSPLTISW3, 9450 OP_VSLDOI4, 9451 OP_VSLDOI8, 9452 OP_VSLDOI12 9453 }; 9454 9455 if (OpNum == OP_COPY) { 9456 if (LHSID == (1*9+2)*9+3) return LHS; 9457 assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!"); 9458 return RHS; 9459 } 9460 9461 SDValue OpLHS, OpRHS; 9462 OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl); 9463 OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl); 9464 9465 int ShufIdxs[16]; 9466 switch (OpNum) { 9467 default: llvm_unreachable("Unknown i32 permute!"); 9468 case OP_VMRGHW: 9469 ShufIdxs[ 0] = 0; ShufIdxs[ 1] = 1; ShufIdxs[ 2] = 2; ShufIdxs[ 3] = 3; 9470 ShufIdxs[ 4] = 16; ShufIdxs[ 5] = 17; ShufIdxs[ 6] = 18; ShufIdxs[ 7] = 19; 9471 ShufIdxs[ 8] = 4; ShufIdxs[ 9] = 5; ShufIdxs[10] = 6; ShufIdxs[11] = 7; 9472 ShufIdxs[12] = 20; ShufIdxs[13] = 21; ShufIdxs[14] = 22; ShufIdxs[15] = 23; 9473 break; 9474 case OP_VMRGLW: 9475 ShufIdxs[ 0] = 8; ShufIdxs[ 1] = 9; ShufIdxs[ 2] = 10; ShufIdxs[ 3] = 11; 9476 ShufIdxs[ 4] = 24; ShufIdxs[ 5] = 25; ShufIdxs[ 6] = 26; ShufIdxs[ 7] = 27; 9477 ShufIdxs[ 8] = 12; ShufIdxs[ 9] = 13; ShufIdxs[10] = 14; ShufIdxs[11] = 15; 9478 ShufIdxs[12] = 28; ShufIdxs[13] = 29; ShufIdxs[14] = 30; ShufIdxs[15] = 31; 9479 break; 9480 case OP_VSPLTISW0: 9481 for (unsigned i = 0; i != 16; ++i) 9482 ShufIdxs[i] = (i&3)+0; 9483 break; 9484 case OP_VSPLTISW1: 9485 for (unsigned i = 0; i != 16; ++i) 9486 ShufIdxs[i] = (i&3)+4; 9487 break; 9488 case OP_VSPLTISW2: 9489 for (unsigned i = 0; i != 16; ++i) 9490 ShufIdxs[i] = (i&3)+8; 9491 break; 9492 case OP_VSPLTISW3: 9493 for (unsigned i = 0; i != 16; ++i) 9494 ShufIdxs[i] = (i&3)+12; 9495 break; 9496 case OP_VSLDOI4: 9497 return BuildVSLDOI(OpLHS, OpRHS, 4, OpLHS.getValueType(), DAG, dl); 9498 case OP_VSLDOI8: 9499 return BuildVSLDOI(OpLHS, OpRHS, 8, OpLHS.getValueType(), DAG, dl); 9500 case OP_VSLDOI12: 9501 return BuildVSLDOI(OpLHS, OpRHS, 12, OpLHS.getValueType(), DAG, dl); 9502 } 9503 EVT VT = OpLHS.getValueType(); 9504 OpLHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OpLHS); 9505 OpRHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OpRHS); 9506 SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, OpLHS, OpRHS, ShufIdxs); 9507 return DAG.getNode(ISD::BITCAST, dl, VT, T); 9508 } 9509 9510 /// lowerToVINSERTB - Return the SDValue if this VECTOR_SHUFFLE can be handled 9511 /// by the VINSERTB instruction introduced in ISA 3.0, else just return default 9512 /// SDValue. 9513 SDValue PPCTargetLowering::lowerToVINSERTB(ShuffleVectorSDNode *N, 9514 SelectionDAG &DAG) const { 9515 const unsigned BytesInVector = 16; 9516 bool IsLE = Subtarget.isLittleEndian(); 9517 SDLoc dl(N); 9518 SDValue V1 = N->getOperand(0); 9519 SDValue V2 = N->getOperand(1); 9520 unsigned ShiftElts = 0, InsertAtByte = 0; 9521 bool Swap = false; 9522 9523 // Shifts required to get the byte we want at element 7. 9524 unsigned LittleEndianShifts[] = {8, 7, 6, 5, 4, 3, 2, 1, 9525 0, 15, 14, 13, 12, 11, 10, 9}; 9526 unsigned BigEndianShifts[] = {9, 10, 11, 12, 13, 14, 15, 0, 9527 1, 2, 3, 4, 5, 6, 7, 8}; 9528 9529 ArrayRef<int> Mask = N->getMask(); 9530 int OriginalOrder[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; 9531 9532 // For each mask element, find out if we're just inserting something 9533 // from V2 into V1 or vice versa. 9534 // Possible permutations inserting an element from V2 into V1: 9535 // X, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 9536 // 0, X, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 9537 // ... 9538 // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, X 9539 // Inserting from V1 into V2 will be similar, except mask range will be 9540 // [16,31]. 9541 9542 bool FoundCandidate = false; 9543 // If both vector operands for the shuffle are the same vector, the mask 9544 // will contain only elements from the first one and the second one will be 9545 // undef. 9546 unsigned VINSERTBSrcElem = IsLE ? 8 : 7; 9547 // Go through the mask of half-words to find an element that's being moved 9548 // from one vector to the other. 9549 for (unsigned i = 0; i < BytesInVector; ++i) { 9550 unsigned CurrentElement = Mask[i]; 9551 // If 2nd operand is undefined, we should only look for element 7 in the 9552 // Mask. 9553 if (V2.isUndef() && CurrentElement != VINSERTBSrcElem) 9554 continue; 9555 9556 bool OtherElementsInOrder = true; 9557 // Examine the other elements in the Mask to see if they're in original 9558 // order. 9559 for (unsigned j = 0; j < BytesInVector; ++j) { 9560 if (j == i) 9561 continue; 9562 // If CurrentElement is from V1 [0,15], then we the rest of the Mask to be 9563 // from V2 [16,31] and vice versa. Unless the 2nd operand is undefined, 9564 // in which we always assume we're always picking from the 1st operand. 9565 int MaskOffset = 9566 (!V2.isUndef() && CurrentElement < BytesInVector) ? BytesInVector : 0; 9567 if (Mask[j] != OriginalOrder[j] + MaskOffset) { 9568 OtherElementsInOrder = false; 9569 break; 9570 } 9571 } 9572 // If other elements are in original order, we record the number of shifts 9573 // we need to get the element we want into element 7. Also record which byte 9574 // in the vector we should insert into. 9575 if (OtherElementsInOrder) { 9576 // If 2nd operand is undefined, we assume no shifts and no swapping. 9577 if (V2.isUndef()) { 9578 ShiftElts = 0; 9579 Swap = false; 9580 } else { 9581 // Only need the last 4-bits for shifts because operands will be swapped if CurrentElement is >= 2^4. 9582 ShiftElts = IsLE ? LittleEndianShifts[CurrentElement & 0xF] 9583 : BigEndianShifts[CurrentElement & 0xF]; 9584 Swap = CurrentElement < BytesInVector; 9585 } 9586 InsertAtByte = IsLE ? BytesInVector - (i + 1) : i; 9587 FoundCandidate = true; 9588 break; 9589 } 9590 } 9591 9592 if (!FoundCandidate) 9593 return SDValue(); 9594 9595 // Candidate found, construct the proper SDAG sequence with VINSERTB, 9596 // optionally with VECSHL if shift is required. 9597 if (Swap) 9598 std::swap(V1, V2); 9599 if (V2.isUndef()) 9600 V2 = V1; 9601 if (ShiftElts) { 9602 SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v16i8, V2, V2, 9603 DAG.getConstant(ShiftElts, dl, MVT::i32)); 9604 return DAG.getNode(PPCISD::VECINSERT, dl, MVT::v16i8, V1, Shl, 9605 DAG.getConstant(InsertAtByte, dl, MVT::i32)); 9606 } 9607 return DAG.getNode(PPCISD::VECINSERT, dl, MVT::v16i8, V1, V2, 9608 DAG.getConstant(InsertAtByte, dl, MVT::i32)); 9609 } 9610 9611 /// lowerToVINSERTH - Return the SDValue if this VECTOR_SHUFFLE can be handled 9612 /// by the VINSERTH instruction introduced in ISA 3.0, else just return default 9613 /// SDValue. 9614 SDValue PPCTargetLowering::lowerToVINSERTH(ShuffleVectorSDNode *N, 9615 SelectionDAG &DAG) const { 9616 const unsigned NumHalfWords = 8; 9617 const unsigned BytesInVector = NumHalfWords * 2; 9618 // Check that the shuffle is on half-words. 9619 if (!isNByteElemShuffleMask(N, 2, 1)) 9620 return SDValue(); 9621 9622 bool IsLE = Subtarget.isLittleEndian(); 9623 SDLoc dl(N); 9624 SDValue V1 = N->getOperand(0); 9625 SDValue V2 = N->getOperand(1); 9626 unsigned ShiftElts = 0, InsertAtByte = 0; 9627 bool Swap = false; 9628 9629 // Shifts required to get the half-word we want at element 3. 9630 unsigned LittleEndianShifts[] = {4, 3, 2, 1, 0, 7, 6, 5}; 9631 unsigned BigEndianShifts[] = {5, 6, 7, 0, 1, 2, 3, 4}; 9632 9633 uint32_t Mask = 0; 9634 uint32_t OriginalOrderLow = 0x1234567; 9635 uint32_t OriginalOrderHigh = 0x89ABCDEF; 9636 // Now we look at mask elements 0,2,4,6,8,10,12,14. Pack the mask into a 9637 // 32-bit space, only need 4-bit nibbles per element. 9638 for (unsigned i = 0; i < NumHalfWords; ++i) { 9639 unsigned MaskShift = (NumHalfWords - 1 - i) * 4; 9640 Mask |= ((uint32_t)(N->getMaskElt(i * 2) / 2) << MaskShift); 9641 } 9642 9643 // For each mask element, find out if we're just inserting something 9644 // from V2 into V1 or vice versa. Possible permutations inserting an element 9645 // from V2 into V1: 9646 // X, 1, 2, 3, 4, 5, 6, 7 9647 // 0, X, 2, 3, 4, 5, 6, 7 9648 // 0, 1, X, 3, 4, 5, 6, 7 9649 // 0, 1, 2, X, 4, 5, 6, 7 9650 // 0, 1, 2, 3, X, 5, 6, 7 9651 // 0, 1, 2, 3, 4, X, 6, 7 9652 // 0, 1, 2, 3, 4, 5, X, 7 9653 // 0, 1, 2, 3, 4, 5, 6, X 9654 // Inserting from V1 into V2 will be similar, except mask range will be [8,15]. 9655 9656 bool FoundCandidate = false; 9657 // Go through the mask of half-words to find an element that's being moved 9658 // from one vector to the other. 9659 for (unsigned i = 0; i < NumHalfWords; ++i) { 9660 unsigned MaskShift = (NumHalfWords - 1 - i) * 4; 9661 uint32_t MaskOneElt = (Mask >> MaskShift) & 0xF; 9662 uint32_t MaskOtherElts = ~(0xF << MaskShift); 9663 uint32_t TargetOrder = 0x0; 9664 9665 // If both vector operands for the shuffle are the same vector, the mask 9666 // will contain only elements from the first one and the second one will be 9667 // undef. 9668 if (V2.isUndef()) { 9669 ShiftElts = 0; 9670 unsigned VINSERTHSrcElem = IsLE ? 4 : 3; 9671 TargetOrder = OriginalOrderLow; 9672 Swap = false; 9673 // Skip if not the correct element or mask of other elements don't equal 9674 // to our expected order. 9675 if (MaskOneElt == VINSERTHSrcElem && 9676 (Mask & MaskOtherElts) == (TargetOrder & MaskOtherElts)) { 9677 InsertAtByte = IsLE ? BytesInVector - (i + 1) * 2 : i * 2; 9678 FoundCandidate = true; 9679 break; 9680 } 9681 } else { // If both operands are defined. 9682 // Target order is [8,15] if the current mask is between [0,7]. 9683 TargetOrder = 9684 (MaskOneElt < NumHalfWords) ? OriginalOrderHigh : OriginalOrderLow; 9685 // Skip if mask of other elements don't equal our expected order. 9686 if ((Mask & MaskOtherElts) == (TargetOrder & MaskOtherElts)) { 9687 // We only need the last 3 bits for the number of shifts. 9688 ShiftElts = IsLE ? LittleEndianShifts[MaskOneElt & 0x7] 9689 : BigEndianShifts[MaskOneElt & 0x7]; 9690 InsertAtByte = IsLE ? BytesInVector - (i + 1) * 2 : i * 2; 9691 Swap = MaskOneElt < NumHalfWords; 9692 FoundCandidate = true; 9693 break; 9694 } 9695 } 9696 } 9697 9698 if (!FoundCandidate) 9699 return SDValue(); 9700 9701 // Candidate found, construct the proper SDAG sequence with VINSERTH, 9702 // optionally with VECSHL if shift is required. 9703 if (Swap) 9704 std::swap(V1, V2); 9705 if (V2.isUndef()) 9706 V2 = V1; 9707 SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); 9708 if (ShiftElts) { 9709 // Double ShiftElts because we're left shifting on v16i8 type. 9710 SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v16i8, V2, V2, 9711 DAG.getConstant(2 * ShiftElts, dl, MVT::i32)); 9712 SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, Shl); 9713 SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v8i16, Conv1, Conv2, 9714 DAG.getConstant(InsertAtByte, dl, MVT::i32)); 9715 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins); 9716 } 9717 SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V2); 9718 SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v8i16, Conv1, Conv2, 9719 DAG.getConstant(InsertAtByte, dl, MVT::i32)); 9720 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins); 9721 } 9722 9723 /// lowerToXXSPLTI32DX - Return the SDValue if this VECTOR_SHUFFLE can be 9724 /// handled by the XXSPLTI32DX instruction introduced in ISA 3.1, otherwise 9725 /// return the default SDValue. 9726 SDValue PPCTargetLowering::lowerToXXSPLTI32DX(ShuffleVectorSDNode *SVN, 9727 SelectionDAG &DAG) const { 9728 // The LHS and RHS may be bitcasts to v16i8 as we canonicalize shuffles 9729 // to v16i8. Peek through the bitcasts to get the actual operands. 9730 SDValue LHS = peekThroughBitcasts(SVN->getOperand(0)); 9731 SDValue RHS = peekThroughBitcasts(SVN->getOperand(1)); 9732 9733 auto ShuffleMask = SVN->getMask(); 9734 SDValue VecShuffle(SVN, 0); 9735 SDLoc DL(SVN); 9736 9737 // Check that we have a four byte shuffle. 9738 if (!isNByteElemShuffleMask(SVN, 4, 1)) 9739 return SDValue(); 9740 9741 // Canonicalize the RHS being a BUILD_VECTOR when lowering to xxsplti32dx. 9742 if (RHS->getOpcode() != ISD::BUILD_VECTOR) { 9743 std::swap(LHS, RHS); 9744 VecShuffle = DAG.getCommutedVectorShuffle(*SVN); 9745 ShuffleMask = cast<ShuffleVectorSDNode>(VecShuffle)->getMask(); 9746 } 9747 9748 // Ensure that the RHS is a vector of constants. 9749 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(RHS.getNode()); 9750 if (!BVN) 9751 return SDValue(); 9752 9753 // Check if RHS is a splat of 4-bytes (or smaller). 9754 APInt APSplatValue, APSplatUndef; 9755 unsigned SplatBitSize; 9756 bool HasAnyUndefs; 9757 if (!BVN->isConstantSplat(APSplatValue, APSplatUndef, SplatBitSize, 9758 HasAnyUndefs, 0, !Subtarget.isLittleEndian()) || 9759 SplatBitSize > 32) 9760 return SDValue(); 9761 9762 // Check that the shuffle mask matches the semantics of XXSPLTI32DX. 9763 // The instruction splats a constant C into two words of the source vector 9764 // producing { C, Unchanged, C, Unchanged } or { Unchanged, C, Unchanged, C }. 9765 // Thus we check that the shuffle mask is the equivalent of 9766 // <0, [4-7], 2, [4-7]> or <[4-7], 1, [4-7], 3> respectively. 9767 // Note: the check above of isNByteElemShuffleMask() ensures that the bytes 9768 // within each word are consecutive, so we only need to check the first byte. 9769 SDValue Index; 9770 bool IsLE = Subtarget.isLittleEndian(); 9771 if ((ShuffleMask[0] == 0 && ShuffleMask[8] == 8) && 9772 (ShuffleMask[4] % 4 == 0 && ShuffleMask[12] % 4 == 0 && 9773 ShuffleMask[4] > 15 && ShuffleMask[12] > 15)) 9774 Index = DAG.getTargetConstant(IsLE ? 0 : 1, DL, MVT::i32); 9775 else if ((ShuffleMask[4] == 4 && ShuffleMask[12] == 12) && 9776 (ShuffleMask[0] % 4 == 0 && ShuffleMask[8] % 4 == 0 && 9777 ShuffleMask[0] > 15 && ShuffleMask[8] > 15)) 9778 Index = DAG.getTargetConstant(IsLE ? 1 : 0, DL, MVT::i32); 9779 else 9780 return SDValue(); 9781 9782 // If the splat is narrower than 32-bits, we need to get the 32-bit value 9783 // for XXSPLTI32DX. 9784 unsigned SplatVal = APSplatValue.getZExtValue(); 9785 for (; SplatBitSize < 32; SplatBitSize <<= 1) 9786 SplatVal |= (SplatVal << SplatBitSize); 9787 9788 SDValue SplatNode = DAG.getNode( 9789 PPCISD::XXSPLTI32DX, DL, MVT::v2i64, DAG.getBitcast(MVT::v2i64, LHS), 9790 Index, DAG.getTargetConstant(SplatVal, DL, MVT::i32)); 9791 return DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, SplatNode); 9792 } 9793 9794 /// LowerROTL - Custom lowering for ROTL(v1i128) to vector_shuffle(v16i8). 9795 /// We lower ROTL(v1i128) to vector_shuffle(v16i8) only if shift amount is 9796 /// a multiple of 8. Otherwise convert it to a scalar rotation(i128) 9797 /// i.e (or (shl x, C1), (srl x, 128-C1)). 9798 SDValue PPCTargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const { 9799 assert(Op.getOpcode() == ISD::ROTL && "Should only be called for ISD::ROTL"); 9800 assert(Op.getValueType() == MVT::v1i128 && 9801 "Only set v1i128 as custom, other type shouldn't reach here!"); 9802 SDLoc dl(Op); 9803 SDValue N0 = peekThroughBitcasts(Op.getOperand(0)); 9804 SDValue N1 = peekThroughBitcasts(Op.getOperand(1)); 9805 unsigned SHLAmt = N1.getConstantOperandVal(0); 9806 if (SHLAmt % 8 == 0) { 9807 SmallVector<int, 16> Mask(16, 0); 9808 std::iota(Mask.begin(), Mask.end(), 0); 9809 std::rotate(Mask.begin(), Mask.begin() + SHLAmt / 8, Mask.end()); 9810 if (SDValue Shuffle = 9811 DAG.getVectorShuffle(MVT::v16i8, dl, DAG.getBitcast(MVT::v16i8, N0), 9812 DAG.getUNDEF(MVT::v16i8), Mask)) 9813 return DAG.getNode(ISD::BITCAST, dl, MVT::v1i128, Shuffle); 9814 } 9815 SDValue ArgVal = DAG.getBitcast(MVT::i128, N0); 9816 SDValue SHLOp = DAG.getNode(ISD::SHL, dl, MVT::i128, ArgVal, 9817 DAG.getConstant(SHLAmt, dl, MVT::i32)); 9818 SDValue SRLOp = DAG.getNode(ISD::SRL, dl, MVT::i128, ArgVal, 9819 DAG.getConstant(128 - SHLAmt, dl, MVT::i32)); 9820 SDValue OROp = DAG.getNode(ISD::OR, dl, MVT::i128, SHLOp, SRLOp); 9821 return DAG.getNode(ISD::BITCAST, dl, MVT::v1i128, OROp); 9822 } 9823 9824 /// LowerVECTOR_SHUFFLE - Return the code we lower for VECTOR_SHUFFLE. If this 9825 /// is a shuffle we can handle in a single instruction, return it. Otherwise, 9826 /// return the code it can be lowered into. Worst case, it can always be 9827 /// lowered into a vperm. 9828 SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, 9829 SelectionDAG &DAG) const { 9830 SDLoc dl(Op); 9831 SDValue V1 = Op.getOperand(0); 9832 SDValue V2 = Op.getOperand(1); 9833 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op); 9834 9835 // Any nodes that were combined in the target-independent combiner prior 9836 // to vector legalization will not be sent to the target combine. Try to 9837 // combine it here. 9838 if (SDValue NewShuffle = combineVectorShuffle(SVOp, DAG)) { 9839 if (!isa<ShuffleVectorSDNode>(NewShuffle)) 9840 return NewShuffle; 9841 Op = NewShuffle; 9842 SVOp = cast<ShuffleVectorSDNode>(Op); 9843 V1 = Op.getOperand(0); 9844 V2 = Op.getOperand(1); 9845 } 9846 EVT VT = Op.getValueType(); 9847 bool isLittleEndian = Subtarget.isLittleEndian(); 9848 9849 unsigned ShiftElts, InsertAtByte; 9850 bool Swap = false; 9851 9852 // If this is a load-and-splat, we can do that with a single instruction 9853 // in some cases. However if the load has multiple uses, we don't want to 9854 // combine it because that will just produce multiple loads. 9855 bool IsPermutedLoad = false; 9856 const SDValue *InputLoad = getNormalLoadInput(V1, IsPermutedLoad); 9857 if (InputLoad && Subtarget.hasVSX() && V2.isUndef() && 9858 (PPC::isSplatShuffleMask(SVOp, 4) || PPC::isSplatShuffleMask(SVOp, 8)) && 9859 InputLoad->hasOneUse()) { 9860 bool IsFourByte = PPC::isSplatShuffleMask(SVOp, 4); 9861 int SplatIdx = 9862 PPC::getSplatIdxForPPCMnemonics(SVOp, IsFourByte ? 4 : 8, DAG); 9863 9864 // The splat index for permuted loads will be in the left half of the vector 9865 // which is strictly wider than the loaded value by 8 bytes. So we need to 9866 // adjust the splat index to point to the correct address in memory. 9867 if (IsPermutedLoad) { 9868 assert((isLittleEndian || IsFourByte) && 9869 "Unexpected size for permuted load on big endian target"); 9870 SplatIdx += IsFourByte ? 2 : 1; 9871 assert((SplatIdx < (IsFourByte ? 4 : 2)) && 9872 "Splat of a value outside of the loaded memory"); 9873 } 9874 9875 LoadSDNode *LD = cast<LoadSDNode>(*InputLoad); 9876 // For 4-byte load-and-splat, we need Power9. 9877 if ((IsFourByte && Subtarget.hasP9Vector()) || !IsFourByte) { 9878 uint64_t Offset = 0; 9879 if (IsFourByte) 9880 Offset = isLittleEndian ? (3 - SplatIdx) * 4 : SplatIdx * 4; 9881 else 9882 Offset = isLittleEndian ? (1 - SplatIdx) * 8 : SplatIdx * 8; 9883 9884 // If the width of the load is the same as the width of the splat, 9885 // loading with an offset would load the wrong memory. 9886 if (LD->getValueType(0).getSizeInBits() == (IsFourByte ? 32 : 64)) 9887 Offset = 0; 9888 9889 SDValue BasePtr = LD->getBasePtr(); 9890 if (Offset != 0) 9891 BasePtr = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()), 9892 BasePtr, DAG.getIntPtrConstant(Offset, dl)); 9893 SDValue Ops[] = { 9894 LD->getChain(), // Chain 9895 BasePtr, // BasePtr 9896 DAG.getValueType(Op.getValueType()) // VT 9897 }; 9898 SDVTList VTL = 9899 DAG.getVTList(IsFourByte ? MVT::v4i32 : MVT::v2i64, MVT::Other); 9900 SDValue LdSplt = 9901 DAG.getMemIntrinsicNode(PPCISD::LD_SPLAT, dl, VTL, 9902 Ops, LD->getMemoryVT(), LD->getMemOperand()); 9903 DAG.ReplaceAllUsesOfValueWith(InputLoad->getValue(1), LdSplt.getValue(1)); 9904 if (LdSplt.getValueType() != SVOp->getValueType(0)) 9905 LdSplt = DAG.getBitcast(SVOp->getValueType(0), LdSplt); 9906 return LdSplt; 9907 } 9908 } 9909 if (Subtarget.hasP9Vector() && 9910 PPC::isXXINSERTWMask(SVOp, ShiftElts, InsertAtByte, Swap, 9911 isLittleEndian)) { 9912 if (Swap) 9913 std::swap(V1, V2); 9914 SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); 9915 SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V2); 9916 if (ShiftElts) { 9917 SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v4i32, Conv2, Conv2, 9918 DAG.getConstant(ShiftElts, dl, MVT::i32)); 9919 SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v4i32, Conv1, Shl, 9920 DAG.getConstant(InsertAtByte, dl, MVT::i32)); 9921 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins); 9922 } 9923 SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v4i32, Conv1, Conv2, 9924 DAG.getConstant(InsertAtByte, dl, MVT::i32)); 9925 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins); 9926 } 9927 9928 if (Subtarget.hasPrefixInstrs()) { 9929 SDValue SplatInsertNode; 9930 if ((SplatInsertNode = lowerToXXSPLTI32DX(SVOp, DAG))) 9931 return SplatInsertNode; 9932 } 9933 9934 if (Subtarget.hasP9Altivec()) { 9935 SDValue NewISDNode; 9936 if ((NewISDNode = lowerToVINSERTH(SVOp, DAG))) 9937 return NewISDNode; 9938 9939 if ((NewISDNode = lowerToVINSERTB(SVOp, DAG))) 9940 return NewISDNode; 9941 } 9942 9943 if (Subtarget.hasVSX() && 9944 PPC::isXXSLDWIShuffleMask(SVOp, ShiftElts, Swap, isLittleEndian)) { 9945 if (Swap) 9946 std::swap(V1, V2); 9947 SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); 9948 SDValue Conv2 = 9949 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V2.isUndef() ? V1 : V2); 9950 9951 SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v4i32, Conv1, Conv2, 9952 DAG.getConstant(ShiftElts, dl, MVT::i32)); 9953 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Shl); 9954 } 9955 9956 if (Subtarget.hasVSX() && 9957 PPC::isXXPERMDIShuffleMask(SVOp, ShiftElts, Swap, isLittleEndian)) { 9958 if (Swap) 9959 std::swap(V1, V2); 9960 SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1); 9961 SDValue Conv2 = 9962 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V2.isUndef() ? V1 : V2); 9963 9964 SDValue PermDI = DAG.getNode(PPCISD::XXPERMDI, dl, MVT::v2i64, Conv1, Conv2, 9965 DAG.getConstant(ShiftElts, dl, MVT::i32)); 9966 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, PermDI); 9967 } 9968 9969 if (Subtarget.hasP9Vector()) { 9970 if (PPC::isXXBRHShuffleMask(SVOp)) { 9971 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); 9972 SDValue ReveHWord = DAG.getNode(ISD::BSWAP, dl, MVT::v8i16, Conv); 9973 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveHWord); 9974 } else if (PPC::isXXBRWShuffleMask(SVOp)) { 9975 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); 9976 SDValue ReveWord = DAG.getNode(ISD::BSWAP, dl, MVT::v4i32, Conv); 9977 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveWord); 9978 } else if (PPC::isXXBRDShuffleMask(SVOp)) { 9979 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1); 9980 SDValue ReveDWord = DAG.getNode(ISD::BSWAP, dl, MVT::v2i64, Conv); 9981 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveDWord); 9982 } else if (PPC::isXXBRQShuffleMask(SVOp)) { 9983 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v1i128, V1); 9984 SDValue ReveQWord = DAG.getNode(ISD::BSWAP, dl, MVT::v1i128, Conv); 9985 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveQWord); 9986 } 9987 } 9988 9989 if (Subtarget.hasVSX()) { 9990 if (V2.isUndef() && PPC::isSplatShuffleMask(SVOp, 4)) { 9991 int SplatIdx = PPC::getSplatIdxForPPCMnemonics(SVOp, 4, DAG); 9992 9993 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); 9994 SDValue Splat = DAG.getNode(PPCISD::XXSPLT, dl, MVT::v4i32, Conv, 9995 DAG.getConstant(SplatIdx, dl, MVT::i32)); 9996 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Splat); 9997 } 9998 9999 // Left shifts of 8 bytes are actually swaps. Convert accordingly. 10000 if (V2.isUndef() && PPC::isVSLDOIShuffleMask(SVOp, 1, DAG) == 8) { 10001 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, V1); 10002 SDValue Swap = DAG.getNode(PPCISD::SWAP_NO_CHAIN, dl, MVT::v2f64, Conv); 10003 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Swap); 10004 } 10005 } 10006 10007 // Cases that are handled by instructions that take permute immediates 10008 // (such as vsplt*) should be left as VECTOR_SHUFFLE nodes so they can be 10009 // selected by the instruction selector. 10010 if (V2.isUndef()) { 10011 if (PPC::isSplatShuffleMask(SVOp, 1) || 10012 PPC::isSplatShuffleMask(SVOp, 2) || 10013 PPC::isSplatShuffleMask(SVOp, 4) || 10014 PPC::isVPKUWUMShuffleMask(SVOp, 1, DAG) || 10015 PPC::isVPKUHUMShuffleMask(SVOp, 1, DAG) || 10016 PPC::isVSLDOIShuffleMask(SVOp, 1, DAG) != -1 || 10017 PPC::isVMRGLShuffleMask(SVOp, 1, 1, DAG) || 10018 PPC::isVMRGLShuffleMask(SVOp, 2, 1, DAG) || 10019 PPC::isVMRGLShuffleMask(SVOp, 4, 1, DAG) || 10020 PPC::isVMRGHShuffleMask(SVOp, 1, 1, DAG) || 10021 PPC::isVMRGHShuffleMask(SVOp, 2, 1, DAG) || 10022 PPC::isVMRGHShuffleMask(SVOp, 4, 1, DAG) || 10023 (Subtarget.hasP8Altivec() && ( 10024 PPC::isVPKUDUMShuffleMask(SVOp, 1, DAG) || 10025 PPC::isVMRGEOShuffleMask(SVOp, true, 1, DAG) || 10026 PPC::isVMRGEOShuffleMask(SVOp, false, 1, DAG)))) { 10027 return Op; 10028 } 10029 } 10030 10031 // Altivec has a variety of "shuffle immediates" that take two vector inputs 10032 // and produce a fixed permutation. If any of these match, do not lower to 10033 // VPERM. 10034 unsigned int ShuffleKind = isLittleEndian ? 2 : 0; 10035 if (PPC::isVPKUWUMShuffleMask(SVOp, ShuffleKind, DAG) || 10036 PPC::isVPKUHUMShuffleMask(SVOp, ShuffleKind, DAG) || 10037 PPC::isVSLDOIShuffleMask(SVOp, ShuffleKind, DAG) != -1 || 10038 PPC::isVMRGLShuffleMask(SVOp, 1, ShuffleKind, DAG) || 10039 PPC::isVMRGLShuffleMask(SVOp, 2, ShuffleKind, DAG) || 10040 PPC::isVMRGLShuffleMask(SVOp, 4, ShuffleKind, DAG) || 10041 PPC::isVMRGHShuffleMask(SVOp, 1, ShuffleKind, DAG) || 10042 PPC::isVMRGHShuffleMask(SVOp, 2, ShuffleKind, DAG) || 10043 PPC::isVMRGHShuffleMask(SVOp, 4, ShuffleKind, DAG) || 10044 (Subtarget.hasP8Altivec() && ( 10045 PPC::isVPKUDUMShuffleMask(SVOp, ShuffleKind, DAG) || 10046 PPC::isVMRGEOShuffleMask(SVOp, true, ShuffleKind, DAG) || 10047 PPC::isVMRGEOShuffleMask(SVOp, false, ShuffleKind, DAG)))) 10048 return Op; 10049 10050 // Check to see if this is a shuffle of 4-byte values. If so, we can use our 10051 // perfect shuffle table to emit an optimal matching sequence. 10052 ArrayRef<int> PermMask = SVOp->getMask(); 10053 10054 unsigned PFIndexes[4]; 10055 bool isFourElementShuffle = true; 10056 for (unsigned i = 0; i != 4 && isFourElementShuffle; ++i) { // Element number 10057 unsigned EltNo = 8; // Start out undef. 10058 for (unsigned j = 0; j != 4; ++j) { // Intra-element byte. 10059 if (PermMask[i*4+j] < 0) 10060 continue; // Undef, ignore it. 10061 10062 unsigned ByteSource = PermMask[i*4+j]; 10063 if ((ByteSource & 3) != j) { 10064 isFourElementShuffle = false; 10065 break; 10066 } 10067 10068 if (EltNo == 8) { 10069 EltNo = ByteSource/4; 10070 } else if (EltNo != ByteSource/4) { 10071 isFourElementShuffle = false; 10072 break; 10073 } 10074 } 10075 PFIndexes[i] = EltNo; 10076 } 10077 10078 // If this shuffle can be expressed as a shuffle of 4-byte elements, use the 10079 // perfect shuffle vector to determine if it is cost effective to do this as 10080 // discrete instructions, or whether we should use a vperm. 10081 // For now, we skip this for little endian until such time as we have a 10082 // little-endian perfect shuffle table. 10083 if (isFourElementShuffle && !isLittleEndian) { 10084 // Compute the index in the perfect shuffle table. 10085 unsigned PFTableIndex = 10086 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3]; 10087 10088 unsigned PFEntry = PerfectShuffleTable[PFTableIndex]; 10089 unsigned Cost = (PFEntry >> 30); 10090 10091 // Determining when to avoid vperm is tricky. Many things affect the cost 10092 // of vperm, particularly how many times the perm mask needs to be computed. 10093 // For example, if the perm mask can be hoisted out of a loop or is already 10094 // used (perhaps because there are multiple permutes with the same shuffle 10095 // mask?) the vperm has a cost of 1. OTOH, hoisting the permute mask out of 10096 // the loop requires an extra register. 10097 // 10098 // As a compromise, we only emit discrete instructions if the shuffle can be 10099 // generated in 3 or fewer operations. When we have loop information 10100 // available, if this block is within a loop, we should avoid using vperm 10101 // for 3-operation perms and use a constant pool load instead. 10102 if (Cost < 3) 10103 return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl); 10104 } 10105 10106 // Lower this to a VPERM(V1, V2, V3) expression, where V3 is a constant 10107 // vector that will get spilled to the constant pool. 10108 if (V2.isUndef()) V2 = V1; 10109 10110 // The SHUFFLE_VECTOR mask is almost exactly what we want for vperm, except 10111 // that it is in input element units, not in bytes. Convert now. 10112 10113 // For little endian, the order of the input vectors is reversed, and 10114 // the permutation mask is complemented with respect to 31. This is 10115 // necessary to produce proper semantics with the big-endian-biased vperm 10116 // instruction. 10117 EVT EltVT = V1.getValueType().getVectorElementType(); 10118 unsigned BytesPerElement = EltVT.getSizeInBits()/8; 10119 10120 SmallVector<SDValue, 16> ResultMask; 10121 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) { 10122 unsigned SrcElt = PermMask[i] < 0 ? 0 : PermMask[i]; 10123 10124 for (unsigned j = 0; j != BytesPerElement; ++j) 10125 if (isLittleEndian) 10126 ResultMask.push_back(DAG.getConstant(31 - (SrcElt*BytesPerElement + j), 10127 dl, MVT::i32)); 10128 else 10129 ResultMask.push_back(DAG.getConstant(SrcElt*BytesPerElement + j, dl, 10130 MVT::i32)); 10131 } 10132 10133 ShufflesHandledWithVPERM++; 10134 SDValue VPermMask = DAG.getBuildVector(MVT::v16i8, dl, ResultMask); 10135 LLVM_DEBUG(dbgs() << "Emitting a VPERM for the following shuffle:\n"); 10136 LLVM_DEBUG(SVOp->dump()); 10137 LLVM_DEBUG(dbgs() << "With the following permute control vector:\n"); 10138 LLVM_DEBUG(VPermMask.dump()); 10139 10140 if (isLittleEndian) 10141 return DAG.getNode(PPCISD::VPERM, dl, V1.getValueType(), 10142 V2, V1, VPermMask); 10143 else 10144 return DAG.getNode(PPCISD::VPERM, dl, V1.getValueType(), 10145 V1, V2, VPermMask); 10146 } 10147 10148 /// getVectorCompareInfo - Given an intrinsic, return false if it is not a 10149 /// vector comparison. If it is, return true and fill in Opc/isDot with 10150 /// information about the intrinsic. 10151 static bool getVectorCompareInfo(SDValue Intrin, int &CompareOpc, 10152 bool &isDot, const PPCSubtarget &Subtarget) { 10153 unsigned IntrinsicID = 10154 cast<ConstantSDNode>(Intrin.getOperand(0))->getZExtValue(); 10155 CompareOpc = -1; 10156 isDot = false; 10157 switch (IntrinsicID) { 10158 default: 10159 return false; 10160 // Comparison predicates. 10161 case Intrinsic::ppc_altivec_vcmpbfp_p: 10162 CompareOpc = 966; 10163 isDot = true; 10164 break; 10165 case Intrinsic::ppc_altivec_vcmpeqfp_p: 10166 CompareOpc = 198; 10167 isDot = true; 10168 break; 10169 case Intrinsic::ppc_altivec_vcmpequb_p: 10170 CompareOpc = 6; 10171 isDot = true; 10172 break; 10173 case Intrinsic::ppc_altivec_vcmpequh_p: 10174 CompareOpc = 70; 10175 isDot = true; 10176 break; 10177 case Intrinsic::ppc_altivec_vcmpequw_p: 10178 CompareOpc = 134; 10179 isDot = true; 10180 break; 10181 case Intrinsic::ppc_altivec_vcmpequd_p: 10182 if (Subtarget.hasVSX() || Subtarget.hasP8Altivec()) { 10183 CompareOpc = 199; 10184 isDot = true; 10185 } else 10186 return false; 10187 break; 10188 case Intrinsic::ppc_altivec_vcmpneb_p: 10189 case Intrinsic::ppc_altivec_vcmpneh_p: 10190 case Intrinsic::ppc_altivec_vcmpnew_p: 10191 case Intrinsic::ppc_altivec_vcmpnezb_p: 10192 case Intrinsic::ppc_altivec_vcmpnezh_p: 10193 case Intrinsic::ppc_altivec_vcmpnezw_p: 10194 if (Subtarget.hasP9Altivec()) { 10195 switch (IntrinsicID) { 10196 default: 10197 llvm_unreachable("Unknown comparison intrinsic."); 10198 case Intrinsic::ppc_altivec_vcmpneb_p: 10199 CompareOpc = 7; 10200 break; 10201 case Intrinsic::ppc_altivec_vcmpneh_p: 10202 CompareOpc = 71; 10203 break; 10204 case Intrinsic::ppc_altivec_vcmpnew_p: 10205 CompareOpc = 135; 10206 break; 10207 case Intrinsic::ppc_altivec_vcmpnezb_p: 10208 CompareOpc = 263; 10209 break; 10210 case Intrinsic::ppc_altivec_vcmpnezh_p: 10211 CompareOpc = 327; 10212 break; 10213 case Intrinsic::ppc_altivec_vcmpnezw_p: 10214 CompareOpc = 391; 10215 break; 10216 } 10217 isDot = true; 10218 } else 10219 return false; 10220 break; 10221 case Intrinsic::ppc_altivec_vcmpgefp_p: 10222 CompareOpc = 454; 10223 isDot = true; 10224 break; 10225 case Intrinsic::ppc_altivec_vcmpgtfp_p: 10226 CompareOpc = 710; 10227 isDot = true; 10228 break; 10229 case Intrinsic::ppc_altivec_vcmpgtsb_p: 10230 CompareOpc = 774; 10231 isDot = true; 10232 break; 10233 case Intrinsic::ppc_altivec_vcmpgtsh_p: 10234 CompareOpc = 838; 10235 isDot = true; 10236 break; 10237 case Intrinsic::ppc_altivec_vcmpgtsw_p: 10238 CompareOpc = 902; 10239 isDot = true; 10240 break; 10241 case Intrinsic::ppc_altivec_vcmpgtsd_p: 10242 if (Subtarget.hasVSX() || Subtarget.hasP8Altivec()) { 10243 CompareOpc = 967; 10244 isDot = true; 10245 } else 10246 return false; 10247 break; 10248 case Intrinsic::ppc_altivec_vcmpgtub_p: 10249 CompareOpc = 518; 10250 isDot = true; 10251 break; 10252 case Intrinsic::ppc_altivec_vcmpgtuh_p: 10253 CompareOpc = 582; 10254 isDot = true; 10255 break; 10256 case Intrinsic::ppc_altivec_vcmpgtuw_p: 10257 CompareOpc = 646; 10258 isDot = true; 10259 break; 10260 case Intrinsic::ppc_altivec_vcmpgtud_p: 10261 if (Subtarget.hasVSX() || Subtarget.hasP8Altivec()) { 10262 CompareOpc = 711; 10263 isDot = true; 10264 } else 10265 return false; 10266 break; 10267 10268 case Intrinsic::ppc_altivec_vcmpequq: 10269 case Intrinsic::ppc_altivec_vcmpgtsq: 10270 case Intrinsic::ppc_altivec_vcmpgtuq: 10271 if (!Subtarget.isISA3_1()) 10272 return false; 10273 switch (IntrinsicID) { 10274 default: 10275 llvm_unreachable("Unknown comparison intrinsic."); 10276 case Intrinsic::ppc_altivec_vcmpequq: 10277 CompareOpc = 455; 10278 break; 10279 case Intrinsic::ppc_altivec_vcmpgtsq: 10280 CompareOpc = 903; 10281 break; 10282 case Intrinsic::ppc_altivec_vcmpgtuq: 10283 CompareOpc = 647; 10284 break; 10285 } 10286 break; 10287 10288 // VSX predicate comparisons use the same infrastructure 10289 case Intrinsic::ppc_vsx_xvcmpeqdp_p: 10290 case Intrinsic::ppc_vsx_xvcmpgedp_p: 10291 case Intrinsic::ppc_vsx_xvcmpgtdp_p: 10292 case Intrinsic::ppc_vsx_xvcmpeqsp_p: 10293 case Intrinsic::ppc_vsx_xvcmpgesp_p: 10294 case Intrinsic::ppc_vsx_xvcmpgtsp_p: 10295 if (Subtarget.hasVSX()) { 10296 switch (IntrinsicID) { 10297 case Intrinsic::ppc_vsx_xvcmpeqdp_p: 10298 CompareOpc = 99; 10299 break; 10300 case Intrinsic::ppc_vsx_xvcmpgedp_p: 10301 CompareOpc = 115; 10302 break; 10303 case Intrinsic::ppc_vsx_xvcmpgtdp_p: 10304 CompareOpc = 107; 10305 break; 10306 case Intrinsic::ppc_vsx_xvcmpeqsp_p: 10307 CompareOpc = 67; 10308 break; 10309 case Intrinsic::ppc_vsx_xvcmpgesp_p: 10310 CompareOpc = 83; 10311 break; 10312 case Intrinsic::ppc_vsx_xvcmpgtsp_p: 10313 CompareOpc = 75; 10314 break; 10315 } 10316 isDot = true; 10317 } else 10318 return false; 10319 break; 10320 10321 // Normal Comparisons. 10322 case Intrinsic::ppc_altivec_vcmpbfp: 10323 CompareOpc = 966; 10324 break; 10325 case Intrinsic::ppc_altivec_vcmpeqfp: 10326 CompareOpc = 198; 10327 break; 10328 case Intrinsic::ppc_altivec_vcmpequb: 10329 CompareOpc = 6; 10330 break; 10331 case Intrinsic::ppc_altivec_vcmpequh: 10332 CompareOpc = 70; 10333 break; 10334 case Intrinsic::ppc_altivec_vcmpequw: 10335 CompareOpc = 134; 10336 break; 10337 case Intrinsic::ppc_altivec_vcmpequd: 10338 if (Subtarget.hasP8Altivec()) 10339 CompareOpc = 199; 10340 else 10341 return false; 10342 break; 10343 case Intrinsic::ppc_altivec_vcmpneb: 10344 case Intrinsic::ppc_altivec_vcmpneh: 10345 case Intrinsic::ppc_altivec_vcmpnew: 10346 case Intrinsic::ppc_altivec_vcmpnezb: 10347 case Intrinsic::ppc_altivec_vcmpnezh: 10348 case Intrinsic::ppc_altivec_vcmpnezw: 10349 if (Subtarget.hasP9Altivec()) 10350 switch (IntrinsicID) { 10351 default: 10352 llvm_unreachable("Unknown comparison intrinsic."); 10353 case Intrinsic::ppc_altivec_vcmpneb: 10354 CompareOpc = 7; 10355 break; 10356 case Intrinsic::ppc_altivec_vcmpneh: 10357 CompareOpc = 71; 10358 break; 10359 case Intrinsic::ppc_altivec_vcmpnew: 10360 CompareOpc = 135; 10361 break; 10362 case Intrinsic::ppc_altivec_vcmpnezb: 10363 CompareOpc = 263; 10364 break; 10365 case Intrinsic::ppc_altivec_vcmpnezh: 10366 CompareOpc = 327; 10367 break; 10368 case Intrinsic::ppc_altivec_vcmpnezw: 10369 CompareOpc = 391; 10370 break; 10371 } 10372 else 10373 return false; 10374 break; 10375 case Intrinsic::ppc_altivec_vcmpgefp: 10376 CompareOpc = 454; 10377 break; 10378 case Intrinsic::ppc_altivec_vcmpgtfp: 10379 CompareOpc = 710; 10380 break; 10381 case Intrinsic::ppc_altivec_vcmpgtsb: 10382 CompareOpc = 774; 10383 break; 10384 case Intrinsic::ppc_altivec_vcmpgtsh: 10385 CompareOpc = 838; 10386 break; 10387 case Intrinsic::ppc_altivec_vcmpgtsw: 10388 CompareOpc = 902; 10389 break; 10390 case Intrinsic::ppc_altivec_vcmpgtsd: 10391 if (Subtarget.hasP8Altivec()) 10392 CompareOpc = 967; 10393 else 10394 return false; 10395 break; 10396 case Intrinsic::ppc_altivec_vcmpgtub: 10397 CompareOpc = 518; 10398 break; 10399 case Intrinsic::ppc_altivec_vcmpgtuh: 10400 CompareOpc = 582; 10401 break; 10402 case Intrinsic::ppc_altivec_vcmpgtuw: 10403 CompareOpc = 646; 10404 break; 10405 case Intrinsic::ppc_altivec_vcmpgtud: 10406 if (Subtarget.hasP8Altivec()) 10407 CompareOpc = 711; 10408 else 10409 return false; 10410 break; 10411 case Intrinsic::ppc_altivec_vcmpequq_p: 10412 case Intrinsic::ppc_altivec_vcmpgtsq_p: 10413 case Intrinsic::ppc_altivec_vcmpgtuq_p: 10414 if (!Subtarget.isISA3_1()) 10415 return false; 10416 switch (IntrinsicID) { 10417 default: 10418 llvm_unreachable("Unknown comparison intrinsic."); 10419 case Intrinsic::ppc_altivec_vcmpequq_p: 10420 CompareOpc = 455; 10421 break; 10422 case Intrinsic::ppc_altivec_vcmpgtsq_p: 10423 CompareOpc = 903; 10424 break; 10425 case Intrinsic::ppc_altivec_vcmpgtuq_p: 10426 CompareOpc = 647; 10427 break; 10428 } 10429 isDot = true; 10430 break; 10431 } 10432 return true; 10433 } 10434 10435 /// LowerINTRINSIC_WO_CHAIN - If this is an intrinsic that we want to custom 10436 /// lower, do it, otherwise return null. 10437 SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, 10438 SelectionDAG &DAG) const { 10439 unsigned IntrinsicID = 10440 cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 10441 10442 SDLoc dl(Op); 10443 10444 switch (IntrinsicID) { 10445 case Intrinsic::thread_pointer: 10446 // Reads the thread pointer register, used for __builtin_thread_pointer. 10447 if (Subtarget.isPPC64()) 10448 return DAG.getRegister(PPC::X13, MVT::i64); 10449 return DAG.getRegister(PPC::R2, MVT::i32); 10450 10451 case Intrinsic::ppc_mma_disassemble_acc: 10452 case Intrinsic::ppc_vsx_disassemble_pair: { 10453 int NumVecs = 2; 10454 SDValue WideVec = Op.getOperand(1); 10455 if (IntrinsicID == Intrinsic::ppc_mma_disassemble_acc) { 10456 NumVecs = 4; 10457 WideVec = DAG.getNode(PPCISD::XXMFACC, dl, MVT::v512i1, WideVec); 10458 } 10459 SmallVector<SDValue, 4> RetOps; 10460 for (int VecNo = 0; VecNo < NumVecs; VecNo++) { 10461 SDValue Extract = DAG.getNode( 10462 PPCISD::EXTRACT_VSX_REG, dl, MVT::v16i8, WideVec, 10463 DAG.getConstant(Subtarget.isLittleEndian() ? NumVecs - 1 - VecNo 10464 : VecNo, 10465 dl, getPointerTy(DAG.getDataLayout()))); 10466 RetOps.push_back(Extract); 10467 } 10468 return DAG.getMergeValues(RetOps, dl); 10469 } 10470 10471 case Intrinsic::ppc_unpack_longdouble: { 10472 auto *Idx = dyn_cast<ConstantSDNode>(Op.getOperand(2)); 10473 assert(Idx && (Idx->getSExtValue() == 0 || Idx->getSExtValue() == 1) && 10474 "Argument of long double unpack must be 0 or 1!"); 10475 return DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::f64, Op.getOperand(1), 10476 DAG.getConstant(!!(Idx->getSExtValue()), dl, 10477 Idx->getValueType(0))); 10478 } 10479 10480 case Intrinsic::ppc_compare_exp_lt: 10481 case Intrinsic::ppc_compare_exp_gt: 10482 case Intrinsic::ppc_compare_exp_eq: 10483 case Intrinsic::ppc_compare_exp_uo: { 10484 unsigned Pred; 10485 switch (IntrinsicID) { 10486 case Intrinsic::ppc_compare_exp_lt: 10487 Pred = PPC::PRED_LT; 10488 break; 10489 case Intrinsic::ppc_compare_exp_gt: 10490 Pred = PPC::PRED_GT; 10491 break; 10492 case Intrinsic::ppc_compare_exp_eq: 10493 Pred = PPC::PRED_EQ; 10494 break; 10495 case Intrinsic::ppc_compare_exp_uo: 10496 Pred = PPC::PRED_UN; 10497 break; 10498 } 10499 return SDValue( 10500 DAG.getMachineNode( 10501 PPC::SELECT_CC_I4, dl, MVT::i32, 10502 {SDValue(DAG.getMachineNode(PPC::XSCMPEXPDP, dl, MVT::i32, 10503 Op.getOperand(1), Op.getOperand(2)), 10504 0), 10505 DAG.getConstant(1, dl, MVT::i32), DAG.getConstant(0, dl, MVT::i32), 10506 DAG.getTargetConstant(Pred, dl, MVT::i32)}), 10507 0); 10508 } 10509 case Intrinsic::ppc_test_data_class_d: 10510 case Intrinsic::ppc_test_data_class_f: { 10511 unsigned CmprOpc = PPC::XSTSTDCDP; 10512 if (IntrinsicID == Intrinsic::ppc_test_data_class_f) 10513 CmprOpc = PPC::XSTSTDCSP; 10514 return SDValue( 10515 DAG.getMachineNode( 10516 PPC::SELECT_CC_I4, dl, MVT::i32, 10517 {SDValue(DAG.getMachineNode(CmprOpc, dl, MVT::i32, Op.getOperand(2), 10518 Op.getOperand(1)), 10519 0), 10520 DAG.getConstant(1, dl, MVT::i32), DAG.getConstant(0, dl, MVT::i32), 10521 DAG.getTargetConstant(PPC::PRED_EQ, dl, MVT::i32)}), 10522 0); 10523 } 10524 case Intrinsic::ppc_convert_f128_to_ppcf128: 10525 case Intrinsic::ppc_convert_ppcf128_to_f128: { 10526 RTLIB::Libcall LC = IntrinsicID == Intrinsic::ppc_convert_ppcf128_to_f128 10527 ? RTLIB::CONVERT_PPCF128_F128 10528 : RTLIB::CONVERT_F128_PPCF128; 10529 MakeLibCallOptions CallOptions; 10530 std::pair<SDValue, SDValue> Result = 10531 makeLibCall(DAG, LC, Op.getValueType(), Op.getOperand(1), CallOptions, 10532 dl, SDValue()); 10533 return Result.first; 10534 } 10535 } 10536 10537 // If this is a lowered altivec predicate compare, CompareOpc is set to the 10538 // opcode number of the comparison. 10539 int CompareOpc; 10540 bool isDot; 10541 if (!getVectorCompareInfo(Op, CompareOpc, isDot, Subtarget)) 10542 return SDValue(); // Don't custom lower most intrinsics. 10543 10544 // If this is a non-dot comparison, make the VCMP node and we are done. 10545 if (!isDot) { 10546 SDValue Tmp = DAG.getNode(PPCISD::VCMP, dl, Op.getOperand(2).getValueType(), 10547 Op.getOperand(1), Op.getOperand(2), 10548 DAG.getConstant(CompareOpc, dl, MVT::i32)); 10549 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Tmp); 10550 } 10551 10552 // Create the PPCISD altivec 'dot' comparison node. 10553 SDValue Ops[] = { 10554 Op.getOperand(2), // LHS 10555 Op.getOperand(3), // RHS 10556 DAG.getConstant(CompareOpc, dl, MVT::i32) 10557 }; 10558 EVT VTs[] = { Op.getOperand(2).getValueType(), MVT::Glue }; 10559 SDValue CompNode = DAG.getNode(PPCISD::VCMP_rec, dl, VTs, Ops); 10560 10561 // Now that we have the comparison, emit a copy from the CR to a GPR. 10562 // This is flagged to the above dot comparison. 10563 SDValue Flags = DAG.getNode(PPCISD::MFOCRF, dl, MVT::i32, 10564 DAG.getRegister(PPC::CR6, MVT::i32), 10565 CompNode.getValue(1)); 10566 10567 // Unpack the result based on how the target uses it. 10568 unsigned BitNo; // Bit # of CR6. 10569 bool InvertBit; // Invert result? 10570 switch (cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue()) { 10571 default: // Can't happen, don't crash on invalid number though. 10572 case 0: // Return the value of the EQ bit of CR6. 10573 BitNo = 0; InvertBit = false; 10574 break; 10575 case 1: // Return the inverted value of the EQ bit of CR6. 10576 BitNo = 0; InvertBit = true; 10577 break; 10578 case 2: // Return the value of the LT bit of CR6. 10579 BitNo = 2; InvertBit = false; 10580 break; 10581 case 3: // Return the inverted value of the LT bit of CR6. 10582 BitNo = 2; InvertBit = true; 10583 break; 10584 } 10585 10586 // Shift the bit into the low position. 10587 Flags = DAG.getNode(ISD::SRL, dl, MVT::i32, Flags, 10588 DAG.getConstant(8 - (3 - BitNo), dl, MVT::i32)); 10589 // Isolate the bit. 10590 Flags = DAG.getNode(ISD::AND, dl, MVT::i32, Flags, 10591 DAG.getConstant(1, dl, MVT::i32)); 10592 10593 // If we are supposed to, toggle the bit. 10594 if (InvertBit) 10595 Flags = DAG.getNode(ISD::XOR, dl, MVT::i32, Flags, 10596 DAG.getConstant(1, dl, MVT::i32)); 10597 return Flags; 10598 } 10599 10600 SDValue PPCTargetLowering::LowerINTRINSIC_VOID(SDValue Op, 10601 SelectionDAG &DAG) const { 10602 // SelectionDAGBuilder::visitTargetIntrinsic may insert one extra chain to 10603 // the beginning of the argument list. 10604 int ArgStart = isa<ConstantSDNode>(Op.getOperand(0)) ? 0 : 1; 10605 SDLoc DL(Op); 10606 switch (cast<ConstantSDNode>(Op.getOperand(ArgStart))->getZExtValue()) { 10607 case Intrinsic::ppc_cfence: { 10608 assert(ArgStart == 1 && "llvm.ppc.cfence must carry a chain argument."); 10609 assert(Subtarget.isPPC64() && "Only 64-bit is supported for now."); 10610 SDValue Val = Op.getOperand(ArgStart + 1); 10611 EVT Ty = Val.getValueType(); 10612 if (Ty == MVT::i128) { 10613 // FIXME: Testing one of two paired registers is sufficient to guarantee 10614 // ordering? 10615 Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, Val); 10616 } 10617 return SDValue( 10618 DAG.getMachineNode(PPC::CFENCE8, DL, MVT::Other, 10619 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Val), 10620 Op.getOperand(0)), 10621 0); 10622 } 10623 default: 10624 break; 10625 } 10626 return SDValue(); 10627 } 10628 10629 // Lower scalar BSWAP64 to xxbrd. 10630 SDValue PPCTargetLowering::LowerBSWAP(SDValue Op, SelectionDAG &DAG) const { 10631 SDLoc dl(Op); 10632 if (!Subtarget.isPPC64()) 10633 return Op; 10634 // MTVSRDD 10635 Op = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2i64, Op.getOperand(0), 10636 Op.getOperand(0)); 10637 // XXBRD 10638 Op = DAG.getNode(ISD::BSWAP, dl, MVT::v2i64, Op); 10639 // MFVSRD 10640 int VectorIndex = 0; 10641 if (Subtarget.isLittleEndian()) 10642 VectorIndex = 1; 10643 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Op, 10644 DAG.getTargetConstant(VectorIndex, dl, MVT::i32)); 10645 return Op; 10646 } 10647 10648 // ATOMIC_CMP_SWAP for i8/i16 needs to zero-extend its input since it will be 10649 // compared to a value that is atomically loaded (atomic loads zero-extend). 10650 SDValue PPCTargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op, 10651 SelectionDAG &DAG) const { 10652 assert(Op.getOpcode() == ISD::ATOMIC_CMP_SWAP && 10653 "Expecting an atomic compare-and-swap here."); 10654 SDLoc dl(Op); 10655 auto *AtomicNode = cast<AtomicSDNode>(Op.getNode()); 10656 EVT MemVT = AtomicNode->getMemoryVT(); 10657 if (MemVT.getSizeInBits() >= 32) 10658 return Op; 10659 10660 SDValue CmpOp = Op.getOperand(2); 10661 // If this is already correctly zero-extended, leave it alone. 10662 auto HighBits = APInt::getHighBitsSet(32, 32 - MemVT.getSizeInBits()); 10663 if (DAG.MaskedValueIsZero(CmpOp, HighBits)) 10664 return Op; 10665 10666 // Clear the high bits of the compare operand. 10667 unsigned MaskVal = (1 << MemVT.getSizeInBits()) - 1; 10668 SDValue NewCmpOp = 10669 DAG.getNode(ISD::AND, dl, MVT::i32, CmpOp, 10670 DAG.getConstant(MaskVal, dl, MVT::i32)); 10671 10672 // Replace the existing compare operand with the properly zero-extended one. 10673 SmallVector<SDValue, 4> Ops; 10674 for (int i = 0, e = AtomicNode->getNumOperands(); i < e; i++) 10675 Ops.push_back(AtomicNode->getOperand(i)); 10676 Ops[2] = NewCmpOp; 10677 MachineMemOperand *MMO = AtomicNode->getMemOperand(); 10678 SDVTList Tys = DAG.getVTList(MVT::i32, MVT::Other); 10679 auto NodeTy = 10680 (MemVT == MVT::i8) ? PPCISD::ATOMIC_CMP_SWAP_8 : PPCISD::ATOMIC_CMP_SWAP_16; 10681 return DAG.getMemIntrinsicNode(NodeTy, dl, Tys, Ops, MemVT, MMO); 10682 } 10683 10684 SDValue PPCTargetLowering::LowerATOMIC_LOAD_STORE(SDValue Op, 10685 SelectionDAG &DAG) const { 10686 AtomicSDNode *N = cast<AtomicSDNode>(Op.getNode()); 10687 EVT MemVT = N->getMemoryVT(); 10688 assert(MemVT.getSimpleVT() == MVT::i128 && 10689 "Expect quadword atomic operations"); 10690 SDLoc dl(N); 10691 unsigned Opc = N->getOpcode(); 10692 switch (Opc) { 10693 case ISD::ATOMIC_LOAD: { 10694 // Lower quadword atomic load to int_ppc_atomic_load_i128 which will be 10695 // lowered to ppc instructions by pattern matching instruction selector. 10696 SDVTList Tys = DAG.getVTList(MVT::i64, MVT::i64, MVT::Other); 10697 SmallVector<SDValue, 4> Ops{ 10698 N->getOperand(0), 10699 DAG.getConstant(Intrinsic::ppc_atomic_load_i128, dl, MVT::i32)}; 10700 for (int I = 1, E = N->getNumOperands(); I < E; ++I) 10701 Ops.push_back(N->getOperand(I)); 10702 SDValue LoadedVal = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, dl, Tys, 10703 Ops, MemVT, N->getMemOperand()); 10704 SDValue ValLo = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i128, LoadedVal); 10705 SDValue ValHi = 10706 DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i128, LoadedVal.getValue(1)); 10707 ValHi = DAG.getNode(ISD::SHL, dl, MVT::i128, ValHi, 10708 DAG.getConstant(64, dl, MVT::i32)); 10709 SDValue Val = 10710 DAG.getNode(ISD::OR, dl, {MVT::i128, MVT::Other}, {ValLo, ValHi}); 10711 return DAG.getNode(ISD::MERGE_VALUES, dl, {MVT::i128, MVT::Other}, 10712 {Val, LoadedVal.getValue(2)}); 10713 } 10714 case ISD::ATOMIC_STORE: { 10715 // Lower quadword atomic store to int_ppc_atomic_store_i128 which will be 10716 // lowered to ppc instructions by pattern matching instruction selector. 10717 SDVTList Tys = DAG.getVTList(MVT::Other); 10718 SmallVector<SDValue, 4> Ops{ 10719 N->getOperand(0), 10720 DAG.getConstant(Intrinsic::ppc_atomic_store_i128, dl, MVT::i32)}; 10721 SDValue Val = N->getOperand(2); 10722 SDValue ValLo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i64, Val); 10723 SDValue ValHi = DAG.getNode(ISD::SRL, dl, MVT::i128, Val, 10724 DAG.getConstant(64, dl, MVT::i32)); 10725 ValHi = DAG.getNode(ISD::TRUNCATE, dl, MVT::i64, ValHi); 10726 Ops.push_back(ValLo); 10727 Ops.push_back(ValHi); 10728 Ops.push_back(N->getOperand(1)); 10729 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, dl, Tys, Ops, MemVT, 10730 N->getMemOperand()); 10731 } 10732 default: 10733 llvm_unreachable("Unexpected atomic opcode"); 10734 } 10735 } 10736 10737 SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op, 10738 SelectionDAG &DAG) const { 10739 SDLoc dl(Op); 10740 // Create a stack slot that is 16-byte aligned. 10741 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); 10742 int FrameIdx = MFI.CreateStackObject(16, Align(16), false); 10743 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 10744 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 10745 10746 // Store the input value into Value#0 of the stack slot. 10747 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), FIdx, 10748 MachinePointerInfo()); 10749 // Load it out. 10750 return DAG.getLoad(Op.getValueType(), dl, Store, FIdx, MachinePointerInfo()); 10751 } 10752 10753 SDValue PPCTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, 10754 SelectionDAG &DAG) const { 10755 assert(Op.getOpcode() == ISD::INSERT_VECTOR_ELT && 10756 "Should only be called for ISD::INSERT_VECTOR_ELT"); 10757 10758 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(2)); 10759 10760 EVT VT = Op.getValueType(); 10761 SDLoc dl(Op); 10762 SDValue V1 = Op.getOperand(0); 10763 SDValue V2 = Op.getOperand(1); 10764 10765 if (VT == MVT::v2f64 && C) 10766 return Op; 10767 10768 if (Subtarget.hasP9Vector()) { 10769 // A f32 load feeding into a v4f32 insert_vector_elt is handled in this way 10770 // because on P10, it allows this specific insert_vector_elt load pattern to 10771 // utilize the refactored load and store infrastructure in order to exploit 10772 // prefixed loads. 10773 // On targets with inexpensive direct moves (Power9 and up), a 10774 // (insert_vector_elt v4f32:$vec, (f32 load)) is always better as an integer 10775 // load since a single precision load will involve conversion to double 10776 // precision on the load followed by another conversion to single precision. 10777 if ((VT == MVT::v4f32) && (V2.getValueType() == MVT::f32) && 10778 (isa<LoadSDNode>(V2))) { 10779 SDValue BitcastVector = DAG.getBitcast(MVT::v4i32, V1); 10780 SDValue BitcastLoad = DAG.getBitcast(MVT::i32, V2); 10781 SDValue InsVecElt = 10782 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v4i32, BitcastVector, 10783 BitcastLoad, Op.getOperand(2)); 10784 return DAG.getBitcast(MVT::v4f32, InsVecElt); 10785 } 10786 } 10787 10788 if (Subtarget.isISA3_1()) { 10789 if ((VT == MVT::v2i64 || VT == MVT::v2f64) && !Subtarget.isPPC64()) 10790 return SDValue(); 10791 // On P10, we have legal lowering for constant and variable indices for 10792 // all vectors. 10793 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 || 10794 VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64) 10795 return Op; 10796 } 10797 10798 // Before P10, we have legal lowering for constant indices but not for 10799 // variable ones. 10800 if (!C) 10801 return SDValue(); 10802 10803 // We can use MTVSRZ + VECINSERT for v8i16 and v16i8 types. 10804 if (VT == MVT::v8i16 || VT == MVT::v16i8) { 10805 SDValue Mtvsrz = DAG.getNode(PPCISD::MTVSRZ, dl, VT, V2); 10806 unsigned BytesInEachElement = VT.getVectorElementType().getSizeInBits() / 8; 10807 unsigned InsertAtElement = C->getZExtValue(); 10808 unsigned InsertAtByte = InsertAtElement * BytesInEachElement; 10809 if (Subtarget.isLittleEndian()) { 10810 InsertAtByte = (16 - BytesInEachElement) - InsertAtByte; 10811 } 10812 return DAG.getNode(PPCISD::VECINSERT, dl, VT, V1, Mtvsrz, 10813 DAG.getConstant(InsertAtByte, dl, MVT::i32)); 10814 } 10815 return Op; 10816 } 10817 10818 SDValue PPCTargetLowering::LowerVectorLoad(SDValue Op, 10819 SelectionDAG &DAG) const { 10820 SDLoc dl(Op); 10821 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode()); 10822 SDValue LoadChain = LN->getChain(); 10823 SDValue BasePtr = LN->getBasePtr(); 10824 EVT VT = Op.getValueType(); 10825 10826 if (VT != MVT::v256i1 && VT != MVT::v512i1) 10827 return Op; 10828 10829 // Type v256i1 is used for pairs and v512i1 is used for accumulators. 10830 // Here we create 2 or 4 v16i8 loads to load the pair or accumulator value in 10831 // 2 or 4 vsx registers. 10832 assert((VT != MVT::v512i1 || Subtarget.hasMMA()) && 10833 "Type unsupported without MMA"); 10834 assert((VT != MVT::v256i1 || Subtarget.pairedVectorMemops()) && 10835 "Type unsupported without paired vector support"); 10836 Align Alignment = LN->getAlign(); 10837 SmallVector<SDValue, 4> Loads; 10838 SmallVector<SDValue, 4> LoadChains; 10839 unsigned NumVecs = VT.getSizeInBits() / 128; 10840 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) { 10841 SDValue Load = 10842 DAG.getLoad(MVT::v16i8, dl, LoadChain, BasePtr, 10843 LN->getPointerInfo().getWithOffset(Idx * 16), 10844 commonAlignment(Alignment, Idx * 16), 10845 LN->getMemOperand()->getFlags(), LN->getAAInfo()); 10846 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 10847 DAG.getConstant(16, dl, BasePtr.getValueType())); 10848 Loads.push_back(Load); 10849 LoadChains.push_back(Load.getValue(1)); 10850 } 10851 if (Subtarget.isLittleEndian()) { 10852 std::reverse(Loads.begin(), Loads.end()); 10853 std::reverse(LoadChains.begin(), LoadChains.end()); 10854 } 10855 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); 10856 SDValue Value = 10857 DAG.getNode(VT == MVT::v512i1 ? PPCISD::ACC_BUILD : PPCISD::PAIR_BUILD, 10858 dl, VT, Loads); 10859 SDValue RetOps[] = {Value, TF}; 10860 return DAG.getMergeValues(RetOps, dl); 10861 } 10862 10863 SDValue PPCTargetLowering::LowerVectorStore(SDValue Op, 10864 SelectionDAG &DAG) const { 10865 SDLoc dl(Op); 10866 StoreSDNode *SN = cast<StoreSDNode>(Op.getNode()); 10867 SDValue StoreChain = SN->getChain(); 10868 SDValue BasePtr = SN->getBasePtr(); 10869 SDValue Value = SN->getValue(); 10870 EVT StoreVT = Value.getValueType(); 10871 10872 if (StoreVT != MVT::v256i1 && StoreVT != MVT::v512i1) 10873 return Op; 10874 10875 // Type v256i1 is used for pairs and v512i1 is used for accumulators. 10876 // Here we create 2 or 4 v16i8 stores to store the pair or accumulator 10877 // underlying registers individually. 10878 assert((StoreVT != MVT::v512i1 || Subtarget.hasMMA()) && 10879 "Type unsupported without MMA"); 10880 assert((StoreVT != MVT::v256i1 || Subtarget.pairedVectorMemops()) && 10881 "Type unsupported without paired vector support"); 10882 Align Alignment = SN->getAlign(); 10883 SmallVector<SDValue, 4> Stores; 10884 unsigned NumVecs = 2; 10885 if (StoreVT == MVT::v512i1) { 10886 Value = DAG.getNode(PPCISD::XXMFACC, dl, MVT::v512i1, Value); 10887 NumVecs = 4; 10888 } 10889 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) { 10890 unsigned VecNum = Subtarget.isLittleEndian() ? NumVecs - 1 - Idx : Idx; 10891 SDValue Elt = DAG.getNode(PPCISD::EXTRACT_VSX_REG, dl, MVT::v16i8, Value, 10892 DAG.getConstant(VecNum, dl, getPointerTy(DAG.getDataLayout()))); 10893 SDValue Store = 10894 DAG.getStore(StoreChain, dl, Elt, BasePtr, 10895 SN->getPointerInfo().getWithOffset(Idx * 16), 10896 commonAlignment(Alignment, Idx * 16), 10897 SN->getMemOperand()->getFlags(), SN->getAAInfo()); 10898 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 10899 DAG.getConstant(16, dl, BasePtr.getValueType())); 10900 Stores.push_back(Store); 10901 } 10902 SDValue TF = DAG.getTokenFactor(dl, Stores); 10903 return TF; 10904 } 10905 10906 SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const { 10907 SDLoc dl(Op); 10908 if (Op.getValueType() == MVT::v4i32) { 10909 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); 10910 10911 SDValue Zero = getCanonicalConstSplat(0, 1, MVT::v4i32, DAG, dl); 10912 // +16 as shift amt. 10913 SDValue Neg16 = getCanonicalConstSplat(-16, 4, MVT::v4i32, DAG, dl); 10914 SDValue RHSSwap = // = vrlw RHS, 16 10915 BuildIntrinsicOp(Intrinsic::ppc_altivec_vrlw, RHS, Neg16, DAG, dl); 10916 10917 // Shrinkify inputs to v8i16. 10918 LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, LHS); 10919 RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, RHS); 10920 RHSSwap = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, RHSSwap); 10921 10922 // Low parts multiplied together, generating 32-bit results (we ignore the 10923 // top parts). 10924 SDValue LoProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmulouh, 10925 LHS, RHS, DAG, dl, MVT::v4i32); 10926 10927 SDValue HiProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmsumuhm, 10928 LHS, RHSSwap, Zero, DAG, dl, MVT::v4i32); 10929 // Shift the high parts up 16 bits. 10930 HiProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vslw, HiProd, 10931 Neg16, DAG, dl); 10932 return DAG.getNode(ISD::ADD, dl, MVT::v4i32, LoProd, HiProd); 10933 } else if (Op.getValueType() == MVT::v16i8) { 10934 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); 10935 bool isLittleEndian = Subtarget.isLittleEndian(); 10936 10937 // Multiply the even 8-bit parts, producing 16-bit sums. 10938 SDValue EvenParts = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmuleub, 10939 LHS, RHS, DAG, dl, MVT::v8i16); 10940 EvenParts = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, EvenParts); 10941 10942 // Multiply the odd 8-bit parts, producing 16-bit sums. 10943 SDValue OddParts = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmuloub, 10944 LHS, RHS, DAG, dl, MVT::v8i16); 10945 OddParts = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OddParts); 10946 10947 // Merge the results together. Because vmuleub and vmuloub are 10948 // instructions with a big-endian bias, we must reverse the 10949 // element numbering and reverse the meaning of "odd" and "even" 10950 // when generating little endian code. 10951 int Ops[16]; 10952 for (unsigned i = 0; i != 8; ++i) { 10953 if (isLittleEndian) { 10954 Ops[i*2 ] = 2*i; 10955 Ops[i*2+1] = 2*i+16; 10956 } else { 10957 Ops[i*2 ] = 2*i+1; 10958 Ops[i*2+1] = 2*i+1+16; 10959 } 10960 } 10961 if (isLittleEndian) 10962 return DAG.getVectorShuffle(MVT::v16i8, dl, OddParts, EvenParts, Ops); 10963 else 10964 return DAG.getVectorShuffle(MVT::v16i8, dl, EvenParts, OddParts, Ops); 10965 } else { 10966 llvm_unreachable("Unknown mul to lower!"); 10967 } 10968 } 10969 10970 SDValue PPCTargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const { 10971 bool IsStrict = Op->isStrictFPOpcode(); 10972 if (Op.getOperand(IsStrict ? 1 : 0).getValueType() == MVT::f128 && 10973 !Subtarget.hasP9Vector()) 10974 return SDValue(); 10975 10976 return Op; 10977 } 10978 10979 // Custom lowering for fpext vf32 to v2f64 10980 SDValue PPCTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const { 10981 10982 assert(Op.getOpcode() == ISD::FP_EXTEND && 10983 "Should only be called for ISD::FP_EXTEND"); 10984 10985 // FIXME: handle extends from half precision float vectors on P9. 10986 // We only want to custom lower an extend from v2f32 to v2f64. 10987 if (Op.getValueType() != MVT::v2f64 || 10988 Op.getOperand(0).getValueType() != MVT::v2f32) 10989 return SDValue(); 10990 10991 SDLoc dl(Op); 10992 SDValue Op0 = Op.getOperand(0); 10993 10994 switch (Op0.getOpcode()) { 10995 default: 10996 return SDValue(); 10997 case ISD::EXTRACT_SUBVECTOR: { 10998 assert(Op0.getNumOperands() == 2 && 10999 isa<ConstantSDNode>(Op0->getOperand(1)) && 11000 "Node should have 2 operands with second one being a constant!"); 11001 11002 if (Op0.getOperand(0).getValueType() != MVT::v4f32) 11003 return SDValue(); 11004 11005 // Custom lower is only done for high or low doubleword. 11006 int Idx = cast<ConstantSDNode>(Op0.getOperand(1))->getZExtValue(); 11007 if (Idx % 2 != 0) 11008 return SDValue(); 11009 11010 // Since input is v4f32, at this point Idx is either 0 or 2. 11011 // Shift to get the doubleword position we want. 11012 int DWord = Idx >> 1; 11013 11014 // High and low word positions are different on little endian. 11015 if (Subtarget.isLittleEndian()) 11016 DWord ^= 0x1; 11017 11018 return DAG.getNode(PPCISD::FP_EXTEND_HALF, dl, MVT::v2f64, 11019 Op0.getOperand(0), DAG.getConstant(DWord, dl, MVT::i32)); 11020 } 11021 case ISD::FADD: 11022 case ISD::FMUL: 11023 case ISD::FSUB: { 11024 SDValue NewLoad[2]; 11025 for (unsigned i = 0, ie = Op0.getNumOperands(); i != ie; ++i) { 11026 // Ensure both input are loads. 11027 SDValue LdOp = Op0.getOperand(i); 11028 if (LdOp.getOpcode() != ISD::LOAD) 11029 return SDValue(); 11030 // Generate new load node. 11031 LoadSDNode *LD = cast<LoadSDNode>(LdOp); 11032 SDValue LoadOps[] = {LD->getChain(), LD->getBasePtr()}; 11033 NewLoad[i] = DAG.getMemIntrinsicNode( 11034 PPCISD::LD_VSX_LH, dl, DAG.getVTList(MVT::v4f32, MVT::Other), LoadOps, 11035 LD->getMemoryVT(), LD->getMemOperand()); 11036 } 11037 SDValue NewOp = 11038 DAG.getNode(Op0.getOpcode(), SDLoc(Op0), MVT::v4f32, NewLoad[0], 11039 NewLoad[1], Op0.getNode()->getFlags()); 11040 return DAG.getNode(PPCISD::FP_EXTEND_HALF, dl, MVT::v2f64, NewOp, 11041 DAG.getConstant(0, dl, MVT::i32)); 11042 } 11043 case ISD::LOAD: { 11044 LoadSDNode *LD = cast<LoadSDNode>(Op0); 11045 SDValue LoadOps[] = {LD->getChain(), LD->getBasePtr()}; 11046 SDValue NewLd = DAG.getMemIntrinsicNode( 11047 PPCISD::LD_VSX_LH, dl, DAG.getVTList(MVT::v4f32, MVT::Other), LoadOps, 11048 LD->getMemoryVT(), LD->getMemOperand()); 11049 return DAG.getNode(PPCISD::FP_EXTEND_HALF, dl, MVT::v2f64, NewLd, 11050 DAG.getConstant(0, dl, MVT::i32)); 11051 } 11052 } 11053 llvm_unreachable("ERROR:Should return for all cases within swtich."); 11054 } 11055 11056 /// LowerOperation - Provide custom lowering hooks for some operations. 11057 /// 11058 SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 11059 switch (Op.getOpcode()) { 11060 default: llvm_unreachable("Wasn't expecting to be able to lower this!"); 11061 case ISD::ConstantPool: return LowerConstantPool(Op, DAG); 11062 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG); 11063 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); 11064 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); 11065 case ISD::JumpTable: return LowerJumpTable(Op, DAG); 11066 case ISD::STRICT_FSETCC: 11067 case ISD::STRICT_FSETCCS: 11068 case ISD::SETCC: return LowerSETCC(Op, DAG); 11069 case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG); 11070 case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG); 11071 11072 case ISD::INLINEASM: 11073 case ISD::INLINEASM_BR: return LowerINLINEASM(Op, DAG); 11074 // Variable argument lowering. 11075 case ISD::VASTART: return LowerVASTART(Op, DAG); 11076 case ISD::VAARG: return LowerVAARG(Op, DAG); 11077 case ISD::VACOPY: return LowerVACOPY(Op, DAG); 11078 11079 case ISD::STACKRESTORE: return LowerSTACKRESTORE(Op, DAG); 11080 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); 11081 case ISD::GET_DYNAMIC_AREA_OFFSET: 11082 return LowerGET_DYNAMIC_AREA_OFFSET(Op, DAG); 11083 11084 // Exception handling lowering. 11085 case ISD::EH_DWARF_CFA: return LowerEH_DWARF_CFA(Op, DAG); 11086 case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG); 11087 case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG); 11088 11089 case ISD::LOAD: return LowerLOAD(Op, DAG); 11090 case ISD::STORE: return LowerSTORE(Op, DAG); 11091 case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG); 11092 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG); 11093 case ISD::STRICT_FP_TO_UINT: 11094 case ISD::STRICT_FP_TO_SINT: 11095 case ISD::FP_TO_UINT: 11096 case ISD::FP_TO_SINT: return LowerFP_TO_INT(Op, DAG, SDLoc(Op)); 11097 case ISD::STRICT_UINT_TO_FP: 11098 case ISD::STRICT_SINT_TO_FP: 11099 case ISD::UINT_TO_FP: 11100 case ISD::SINT_TO_FP: return LowerINT_TO_FP(Op, DAG); 11101 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); 11102 11103 // Lower 64-bit shifts. 11104 case ISD::SHL_PARTS: return LowerSHL_PARTS(Op, DAG); 11105 case ISD::SRL_PARTS: return LowerSRL_PARTS(Op, DAG); 11106 case ISD::SRA_PARTS: return LowerSRA_PARTS(Op, DAG); 11107 11108 case ISD::FSHL: return LowerFunnelShift(Op, DAG); 11109 case ISD::FSHR: return LowerFunnelShift(Op, DAG); 11110 11111 // Vector-related lowering. 11112 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); 11113 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); 11114 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); 11115 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG); 11116 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); 11117 case ISD::MUL: return LowerMUL(Op, DAG); 11118 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG); 11119 case ISD::STRICT_FP_ROUND: 11120 case ISD::FP_ROUND: 11121 return LowerFP_ROUND(Op, DAG); 11122 case ISD::ROTL: return LowerROTL(Op, DAG); 11123 11124 // For counter-based loop handling. 11125 case ISD::INTRINSIC_W_CHAIN: return SDValue(); 11126 11127 case ISD::BITCAST: return LowerBITCAST(Op, DAG); 11128 11129 // Frame & Return address. 11130 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); 11131 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); 11132 11133 case ISD::INTRINSIC_VOID: 11134 return LowerINTRINSIC_VOID(Op, DAG); 11135 case ISD::BSWAP: 11136 return LowerBSWAP(Op, DAG); 11137 case ISD::ATOMIC_CMP_SWAP: 11138 return LowerATOMIC_CMP_SWAP(Op, DAG); 11139 case ISD::ATOMIC_STORE: 11140 return LowerATOMIC_LOAD_STORE(Op, DAG); 11141 } 11142 } 11143 11144 void PPCTargetLowering::ReplaceNodeResults(SDNode *N, 11145 SmallVectorImpl<SDValue>&Results, 11146 SelectionDAG &DAG) const { 11147 SDLoc dl(N); 11148 switch (N->getOpcode()) { 11149 default: 11150 llvm_unreachable("Do not know how to custom type legalize this operation!"); 11151 case ISD::ATOMIC_LOAD: { 11152 SDValue Res = LowerATOMIC_LOAD_STORE(SDValue(N, 0), DAG); 11153 Results.push_back(Res); 11154 Results.push_back(Res.getValue(1)); 11155 break; 11156 } 11157 case ISD::READCYCLECOUNTER: { 11158 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other); 11159 SDValue RTB = DAG.getNode(PPCISD::READ_TIME_BASE, dl, VTs, N->getOperand(0)); 11160 11161 Results.push_back( 11162 DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, RTB, RTB.getValue(1))); 11163 Results.push_back(RTB.getValue(2)); 11164 break; 11165 } 11166 case ISD::INTRINSIC_W_CHAIN: { 11167 if (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() != 11168 Intrinsic::loop_decrement) 11169 break; 11170 11171 assert(N->getValueType(0) == MVT::i1 && 11172 "Unexpected result type for CTR decrement intrinsic"); 11173 EVT SVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), 11174 N->getValueType(0)); 11175 SDVTList VTs = DAG.getVTList(SVT, MVT::Other); 11176 SDValue NewInt = DAG.getNode(N->getOpcode(), dl, VTs, N->getOperand(0), 11177 N->getOperand(1)); 11178 11179 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewInt)); 11180 Results.push_back(NewInt.getValue(1)); 11181 break; 11182 } 11183 case ISD::INTRINSIC_WO_CHAIN: { 11184 switch (cast<ConstantSDNode>(N->getOperand(0))->getZExtValue()) { 11185 case Intrinsic::ppc_pack_longdouble: 11186 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::ppcf128, 11187 N->getOperand(2), N->getOperand(1))); 11188 break; 11189 case Intrinsic::ppc_convert_f128_to_ppcf128: 11190 Results.push_back(LowerINTRINSIC_WO_CHAIN(SDValue(N, 0), DAG)); 11191 break; 11192 } 11193 break; 11194 } 11195 case ISD::VAARG: { 11196 if (!Subtarget.isSVR4ABI() || Subtarget.isPPC64()) 11197 return; 11198 11199 EVT VT = N->getValueType(0); 11200 11201 if (VT == MVT::i64) { 11202 SDValue NewNode = LowerVAARG(SDValue(N, 1), DAG); 11203 11204 Results.push_back(NewNode); 11205 Results.push_back(NewNode.getValue(1)); 11206 } 11207 return; 11208 } 11209 case ISD::STRICT_FP_TO_SINT: 11210 case ISD::STRICT_FP_TO_UINT: 11211 case ISD::FP_TO_SINT: 11212 case ISD::FP_TO_UINT: { 11213 // LowerFP_TO_INT() can only handle f32 and f64. 11214 if (N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType() == 11215 MVT::ppcf128) 11216 return; 11217 SDValue LoweredValue = LowerFP_TO_INT(SDValue(N, 0), DAG, dl); 11218 Results.push_back(LoweredValue); 11219 if (N->isStrictFPOpcode()) 11220 Results.push_back(LoweredValue.getValue(1)); 11221 return; 11222 } 11223 case ISD::TRUNCATE: { 11224 if (!N->getValueType(0).isVector()) 11225 return; 11226 SDValue Lowered = LowerTRUNCATEVector(SDValue(N, 0), DAG); 11227 if (Lowered) 11228 Results.push_back(Lowered); 11229 return; 11230 } 11231 case ISD::FSHL: 11232 case ISD::FSHR: 11233 // Don't handle funnel shifts here. 11234 return; 11235 case ISD::BITCAST: 11236 // Don't handle bitcast here. 11237 return; 11238 case ISD::FP_EXTEND: 11239 SDValue Lowered = LowerFP_EXTEND(SDValue(N, 0), DAG); 11240 if (Lowered) 11241 Results.push_back(Lowered); 11242 return; 11243 } 11244 } 11245 11246 //===----------------------------------------------------------------------===// 11247 // Other Lowering Code 11248 //===----------------------------------------------------------------------===// 11249 11250 static Instruction *callIntrinsic(IRBuilderBase &Builder, Intrinsic::ID Id) { 11251 Module *M = Builder.GetInsertBlock()->getParent()->getParent(); 11252 Function *Func = Intrinsic::getDeclaration(M, Id); 11253 return Builder.CreateCall(Func, {}); 11254 } 11255 11256 // The mappings for emitLeading/TrailingFence is taken from 11257 // http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html 11258 Instruction *PPCTargetLowering::emitLeadingFence(IRBuilderBase &Builder, 11259 Instruction *Inst, 11260 AtomicOrdering Ord) const { 11261 if (Ord == AtomicOrdering::SequentiallyConsistent) 11262 return callIntrinsic(Builder, Intrinsic::ppc_sync); 11263 if (isReleaseOrStronger(Ord)) 11264 return callIntrinsic(Builder, Intrinsic::ppc_lwsync); 11265 return nullptr; 11266 } 11267 11268 Instruction *PPCTargetLowering::emitTrailingFence(IRBuilderBase &Builder, 11269 Instruction *Inst, 11270 AtomicOrdering Ord) const { 11271 if (Inst->hasAtomicLoad() && isAcquireOrStronger(Ord)) { 11272 // See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html and 11273 // http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2011.03.04a.html 11274 // and http://www.cl.cam.ac.uk/~pes20/cppppc/ for justification. 11275 if (isa<LoadInst>(Inst) && Subtarget.isPPC64()) 11276 return Builder.CreateCall( 11277 Intrinsic::getDeclaration( 11278 Builder.GetInsertBlock()->getParent()->getParent(), 11279 Intrinsic::ppc_cfence, {Inst->getType()}), 11280 {Inst}); 11281 // FIXME: Can use isync for rmw operation. 11282 return callIntrinsic(Builder, Intrinsic::ppc_lwsync); 11283 } 11284 return nullptr; 11285 } 11286 11287 MachineBasicBlock * 11288 PPCTargetLowering::EmitAtomicBinary(MachineInstr &MI, MachineBasicBlock *BB, 11289 unsigned AtomicSize, 11290 unsigned BinOpcode, 11291 unsigned CmpOpcode, 11292 unsigned CmpPred) const { 11293 // This also handles ATOMIC_SWAP, indicated by BinOpcode==0. 11294 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 11295 11296 auto LoadMnemonic = PPC::LDARX; 11297 auto StoreMnemonic = PPC::STDCX; 11298 switch (AtomicSize) { 11299 default: 11300 llvm_unreachable("Unexpected size of atomic entity"); 11301 case 1: 11302 LoadMnemonic = PPC::LBARX; 11303 StoreMnemonic = PPC::STBCX; 11304 assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4"); 11305 break; 11306 case 2: 11307 LoadMnemonic = PPC::LHARX; 11308 StoreMnemonic = PPC::STHCX; 11309 assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4"); 11310 break; 11311 case 4: 11312 LoadMnemonic = PPC::LWARX; 11313 StoreMnemonic = PPC::STWCX; 11314 break; 11315 case 8: 11316 LoadMnemonic = PPC::LDARX; 11317 StoreMnemonic = PPC::STDCX; 11318 break; 11319 } 11320 11321 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 11322 MachineFunction *F = BB->getParent(); 11323 MachineFunction::iterator It = ++BB->getIterator(); 11324 11325 Register dest = MI.getOperand(0).getReg(); 11326 Register ptrA = MI.getOperand(1).getReg(); 11327 Register ptrB = MI.getOperand(2).getReg(); 11328 Register incr = MI.getOperand(3).getReg(); 11329 DebugLoc dl = MI.getDebugLoc(); 11330 11331 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(LLVM_BB); 11332 MachineBasicBlock *loop2MBB = 11333 CmpOpcode ? F->CreateMachineBasicBlock(LLVM_BB) : nullptr; 11334 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); 11335 F->insert(It, loopMBB); 11336 if (CmpOpcode) 11337 F->insert(It, loop2MBB); 11338 F->insert(It, exitMBB); 11339 exitMBB->splice(exitMBB->begin(), BB, 11340 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 11341 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 11342 11343 MachineRegisterInfo &RegInfo = F->getRegInfo(); 11344 Register TmpReg = (!BinOpcode) ? incr : 11345 RegInfo.createVirtualRegister( AtomicSize == 8 ? &PPC::G8RCRegClass 11346 : &PPC::GPRCRegClass); 11347 11348 // thisMBB: 11349 // ... 11350 // fallthrough --> loopMBB 11351 BB->addSuccessor(loopMBB); 11352 11353 // loopMBB: 11354 // l[wd]arx dest, ptr 11355 // add r0, dest, incr 11356 // st[wd]cx. r0, ptr 11357 // bne- loopMBB 11358 // fallthrough --> exitMBB 11359 11360 // For max/min... 11361 // loopMBB: 11362 // l[wd]arx dest, ptr 11363 // cmpl?[wd] incr, dest 11364 // bgt exitMBB 11365 // loop2MBB: 11366 // st[wd]cx. dest, ptr 11367 // bne- loopMBB 11368 // fallthrough --> exitMBB 11369 11370 BB = loopMBB; 11371 BuildMI(BB, dl, TII->get(LoadMnemonic), dest) 11372 .addReg(ptrA).addReg(ptrB); 11373 if (BinOpcode) 11374 BuildMI(BB, dl, TII->get(BinOpcode), TmpReg).addReg(incr).addReg(dest); 11375 if (CmpOpcode) { 11376 // Signed comparisons of byte or halfword values must be sign-extended. 11377 if (CmpOpcode == PPC::CMPW && AtomicSize < 4) { 11378 Register ExtReg = RegInfo.createVirtualRegister(&PPC::GPRCRegClass); 11379 BuildMI(BB, dl, TII->get(AtomicSize == 1 ? PPC::EXTSB : PPC::EXTSH), 11380 ExtReg).addReg(dest); 11381 BuildMI(BB, dl, TII->get(CmpOpcode), PPC::CR0) 11382 .addReg(incr).addReg(ExtReg); 11383 } else 11384 BuildMI(BB, dl, TII->get(CmpOpcode), PPC::CR0) 11385 .addReg(incr).addReg(dest); 11386 11387 BuildMI(BB, dl, TII->get(PPC::BCC)) 11388 .addImm(CmpPred).addReg(PPC::CR0).addMBB(exitMBB); 11389 BB->addSuccessor(loop2MBB); 11390 BB->addSuccessor(exitMBB); 11391 BB = loop2MBB; 11392 } 11393 BuildMI(BB, dl, TII->get(StoreMnemonic)) 11394 .addReg(TmpReg).addReg(ptrA).addReg(ptrB); 11395 BuildMI(BB, dl, TII->get(PPC::BCC)) 11396 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(loopMBB); 11397 BB->addSuccessor(loopMBB); 11398 BB->addSuccessor(exitMBB); 11399 11400 // exitMBB: 11401 // ... 11402 BB = exitMBB; 11403 return BB; 11404 } 11405 11406 static bool isSignExtended(MachineInstr &MI, const PPCInstrInfo *TII) { 11407 switch(MI.getOpcode()) { 11408 default: 11409 return false; 11410 case PPC::COPY: 11411 return TII->isSignExtended(MI); 11412 case PPC::LHA: 11413 case PPC::LHA8: 11414 case PPC::LHAU: 11415 case PPC::LHAU8: 11416 case PPC::LHAUX: 11417 case PPC::LHAUX8: 11418 case PPC::LHAX: 11419 case PPC::LHAX8: 11420 case PPC::LWA: 11421 case PPC::LWAUX: 11422 case PPC::LWAX: 11423 case PPC::LWAX_32: 11424 case PPC::LWA_32: 11425 case PPC::PLHA: 11426 case PPC::PLHA8: 11427 case PPC::PLHA8pc: 11428 case PPC::PLHApc: 11429 case PPC::PLWA: 11430 case PPC::PLWA8: 11431 case PPC::PLWA8pc: 11432 case PPC::PLWApc: 11433 case PPC::EXTSB: 11434 case PPC::EXTSB8: 11435 case PPC::EXTSB8_32_64: 11436 case PPC::EXTSB8_rec: 11437 case PPC::EXTSB_rec: 11438 case PPC::EXTSH: 11439 case PPC::EXTSH8: 11440 case PPC::EXTSH8_32_64: 11441 case PPC::EXTSH8_rec: 11442 case PPC::EXTSH_rec: 11443 case PPC::EXTSW: 11444 case PPC::EXTSWSLI: 11445 case PPC::EXTSWSLI_32_64: 11446 case PPC::EXTSWSLI_32_64_rec: 11447 case PPC::EXTSWSLI_rec: 11448 case PPC::EXTSW_32: 11449 case PPC::EXTSW_32_64: 11450 case PPC::EXTSW_32_64_rec: 11451 case PPC::EXTSW_rec: 11452 case PPC::SRAW: 11453 case PPC::SRAWI: 11454 case PPC::SRAWI_rec: 11455 case PPC::SRAW_rec: 11456 return true; 11457 } 11458 return false; 11459 } 11460 11461 MachineBasicBlock *PPCTargetLowering::EmitPartwordAtomicBinary( 11462 MachineInstr &MI, MachineBasicBlock *BB, 11463 bool is8bit, // operation 11464 unsigned BinOpcode, unsigned CmpOpcode, unsigned CmpPred) const { 11465 // This also handles ATOMIC_SWAP, indicated by BinOpcode==0. 11466 const PPCInstrInfo *TII = Subtarget.getInstrInfo(); 11467 11468 // If this is a signed comparison and the value being compared is not known 11469 // to be sign extended, sign extend it here. 11470 DebugLoc dl = MI.getDebugLoc(); 11471 MachineFunction *F = BB->getParent(); 11472 MachineRegisterInfo &RegInfo = F->getRegInfo(); 11473 Register incr = MI.getOperand(3).getReg(); 11474 bool IsSignExtended = Register::isVirtualRegister(incr) && 11475 isSignExtended(*RegInfo.getVRegDef(incr), TII); 11476 11477 if (CmpOpcode == PPC::CMPW && !IsSignExtended) { 11478 Register ValueReg = RegInfo.createVirtualRegister(&PPC::GPRCRegClass); 11479 BuildMI(*BB, MI, dl, TII->get(is8bit ? PPC::EXTSB : PPC::EXTSH), ValueReg) 11480 .addReg(MI.getOperand(3).getReg()); 11481 MI.getOperand(3).setReg(ValueReg); 11482 } 11483 // If we support part-word atomic mnemonics, just use them 11484 if (Subtarget.hasPartwordAtomics()) 11485 return EmitAtomicBinary(MI, BB, is8bit ? 1 : 2, BinOpcode, CmpOpcode, 11486 CmpPred); 11487 11488 // In 64 bit mode we have to use 64 bits for addresses, even though the 11489 // lwarx/stwcx are 32 bits. With the 32-bit atomics we can use address 11490 // registers without caring whether they're 32 or 64, but here we're 11491 // doing actual arithmetic on the addresses. 11492 bool is64bit = Subtarget.isPPC64(); 11493 bool isLittleEndian = Subtarget.isLittleEndian(); 11494 unsigned ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO; 11495 11496 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 11497 MachineFunction::iterator It = ++BB->getIterator(); 11498 11499 Register dest = MI.getOperand(0).getReg(); 11500 Register ptrA = MI.getOperand(1).getReg(); 11501 Register ptrB = MI.getOperand(2).getReg(); 11502 11503 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(LLVM_BB); 11504 MachineBasicBlock *loop2MBB = 11505 CmpOpcode ? F->CreateMachineBasicBlock(LLVM_BB) : nullptr; 11506 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); 11507 F->insert(It, loopMBB); 11508 if (CmpOpcode) 11509 F->insert(It, loop2MBB); 11510 F->insert(It, exitMBB); 11511 exitMBB->splice(exitMBB->begin(), BB, 11512 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 11513 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 11514 11515 const TargetRegisterClass *RC = 11516 is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; 11517 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass; 11518 11519 Register PtrReg = RegInfo.createVirtualRegister(RC); 11520 Register Shift1Reg = RegInfo.createVirtualRegister(GPRC); 11521 Register ShiftReg = 11522 isLittleEndian ? Shift1Reg : RegInfo.createVirtualRegister(GPRC); 11523 Register Incr2Reg = RegInfo.createVirtualRegister(GPRC); 11524 Register MaskReg = RegInfo.createVirtualRegister(GPRC); 11525 Register Mask2Reg = RegInfo.createVirtualRegister(GPRC); 11526 Register Mask3Reg = RegInfo.createVirtualRegister(GPRC); 11527 Register Tmp2Reg = RegInfo.createVirtualRegister(GPRC); 11528 Register Tmp3Reg = RegInfo.createVirtualRegister(GPRC); 11529 Register Tmp4Reg = RegInfo.createVirtualRegister(GPRC); 11530 Register TmpDestReg = RegInfo.createVirtualRegister(GPRC); 11531 Register SrwDestReg = RegInfo.createVirtualRegister(GPRC); 11532 Register Ptr1Reg; 11533 Register TmpReg = 11534 (!BinOpcode) ? Incr2Reg : RegInfo.createVirtualRegister(GPRC); 11535 11536 // thisMBB: 11537 // ... 11538 // fallthrough --> loopMBB 11539 BB->addSuccessor(loopMBB); 11540 11541 // The 4-byte load must be aligned, while a char or short may be 11542 // anywhere in the word. Hence all this nasty bookkeeping code. 11543 // add ptr1, ptrA, ptrB [copy if ptrA==0] 11544 // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27] 11545 // xori shift, shift1, 24 [16] 11546 // rlwinm ptr, ptr1, 0, 0, 29 11547 // slw incr2, incr, shift 11548 // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535] 11549 // slw mask, mask2, shift 11550 // loopMBB: 11551 // lwarx tmpDest, ptr 11552 // add tmp, tmpDest, incr2 11553 // andc tmp2, tmpDest, mask 11554 // and tmp3, tmp, mask 11555 // or tmp4, tmp3, tmp2 11556 // stwcx. tmp4, ptr 11557 // bne- loopMBB 11558 // fallthrough --> exitMBB 11559 // srw SrwDest, tmpDest, shift 11560 // rlwinm SrwDest, SrwDest, 0, 24 [16], 31 11561 if (ptrA != ZeroReg) { 11562 Ptr1Reg = RegInfo.createVirtualRegister(RC); 11563 BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg) 11564 .addReg(ptrA) 11565 .addReg(ptrB); 11566 } else { 11567 Ptr1Reg = ptrB; 11568 } 11569 // We need use 32-bit subregister to avoid mismatch register class in 64-bit 11570 // mode. 11571 BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg) 11572 .addReg(Ptr1Reg, 0, is64bit ? PPC::sub_32 : 0) 11573 .addImm(3) 11574 .addImm(27) 11575 .addImm(is8bit ? 28 : 27); 11576 if (!isLittleEndian) 11577 BuildMI(BB, dl, TII->get(PPC::XORI), ShiftReg) 11578 .addReg(Shift1Reg) 11579 .addImm(is8bit ? 24 : 16); 11580 if (is64bit) 11581 BuildMI(BB, dl, TII->get(PPC::RLDICR), PtrReg) 11582 .addReg(Ptr1Reg) 11583 .addImm(0) 11584 .addImm(61); 11585 else 11586 BuildMI(BB, dl, TII->get(PPC::RLWINM), PtrReg) 11587 .addReg(Ptr1Reg) 11588 .addImm(0) 11589 .addImm(0) 11590 .addImm(29); 11591 BuildMI(BB, dl, TII->get(PPC::SLW), Incr2Reg).addReg(incr).addReg(ShiftReg); 11592 if (is8bit) 11593 BuildMI(BB, dl, TII->get(PPC::LI), Mask2Reg).addImm(255); 11594 else { 11595 BuildMI(BB, dl, TII->get(PPC::LI), Mask3Reg).addImm(0); 11596 BuildMI(BB, dl, TII->get(PPC::ORI), Mask2Reg) 11597 .addReg(Mask3Reg) 11598 .addImm(65535); 11599 } 11600 BuildMI(BB, dl, TII->get(PPC::SLW), MaskReg) 11601 .addReg(Mask2Reg) 11602 .addReg(ShiftReg); 11603 11604 BB = loopMBB; 11605 BuildMI(BB, dl, TII->get(PPC::LWARX), TmpDestReg) 11606 .addReg(ZeroReg) 11607 .addReg(PtrReg); 11608 if (BinOpcode) 11609 BuildMI(BB, dl, TII->get(BinOpcode), TmpReg) 11610 .addReg(Incr2Reg) 11611 .addReg(TmpDestReg); 11612 BuildMI(BB, dl, TII->get(PPC::ANDC), Tmp2Reg) 11613 .addReg(TmpDestReg) 11614 .addReg(MaskReg); 11615 BuildMI(BB, dl, TII->get(PPC::AND), Tmp3Reg).addReg(TmpReg).addReg(MaskReg); 11616 if (CmpOpcode) { 11617 // For unsigned comparisons, we can directly compare the shifted values. 11618 // For signed comparisons we shift and sign extend. 11619 Register SReg = RegInfo.createVirtualRegister(GPRC); 11620 BuildMI(BB, dl, TII->get(PPC::AND), SReg) 11621 .addReg(TmpDestReg) 11622 .addReg(MaskReg); 11623 unsigned ValueReg = SReg; 11624 unsigned CmpReg = Incr2Reg; 11625 if (CmpOpcode == PPC::CMPW) { 11626 ValueReg = RegInfo.createVirtualRegister(GPRC); 11627 BuildMI(BB, dl, TII->get(PPC::SRW), ValueReg) 11628 .addReg(SReg) 11629 .addReg(ShiftReg); 11630 Register ValueSReg = RegInfo.createVirtualRegister(GPRC); 11631 BuildMI(BB, dl, TII->get(is8bit ? PPC::EXTSB : PPC::EXTSH), ValueSReg) 11632 .addReg(ValueReg); 11633 ValueReg = ValueSReg; 11634 CmpReg = incr; 11635 } 11636 BuildMI(BB, dl, TII->get(CmpOpcode), PPC::CR0) 11637 .addReg(CmpReg) 11638 .addReg(ValueReg); 11639 BuildMI(BB, dl, TII->get(PPC::BCC)) 11640 .addImm(CmpPred) 11641 .addReg(PPC::CR0) 11642 .addMBB(exitMBB); 11643 BB->addSuccessor(loop2MBB); 11644 BB->addSuccessor(exitMBB); 11645 BB = loop2MBB; 11646 } 11647 BuildMI(BB, dl, TII->get(PPC::OR), Tmp4Reg).addReg(Tmp3Reg).addReg(Tmp2Reg); 11648 BuildMI(BB, dl, TII->get(PPC::STWCX)) 11649 .addReg(Tmp4Reg) 11650 .addReg(ZeroReg) 11651 .addReg(PtrReg); 11652 BuildMI(BB, dl, TII->get(PPC::BCC)) 11653 .addImm(PPC::PRED_NE) 11654 .addReg(PPC::CR0) 11655 .addMBB(loopMBB); 11656 BB->addSuccessor(loopMBB); 11657 BB->addSuccessor(exitMBB); 11658 11659 // exitMBB: 11660 // ... 11661 BB = exitMBB; 11662 // Since the shift amount is not a constant, we need to clear 11663 // the upper bits with a separate RLWINM. 11664 BuildMI(*BB, BB->begin(), dl, TII->get(PPC::RLWINM), dest) 11665 .addReg(SrwDestReg) 11666 .addImm(0) 11667 .addImm(is8bit ? 24 : 16) 11668 .addImm(31); 11669 BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), SrwDestReg) 11670 .addReg(TmpDestReg) 11671 .addReg(ShiftReg); 11672 return BB; 11673 } 11674 11675 llvm::MachineBasicBlock * 11676 PPCTargetLowering::emitEHSjLjSetJmp(MachineInstr &MI, 11677 MachineBasicBlock *MBB) const { 11678 DebugLoc DL = MI.getDebugLoc(); 11679 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 11680 const PPCRegisterInfo *TRI = Subtarget.getRegisterInfo(); 11681 11682 MachineFunction *MF = MBB->getParent(); 11683 MachineRegisterInfo &MRI = MF->getRegInfo(); 11684 11685 const BasicBlock *BB = MBB->getBasicBlock(); 11686 MachineFunction::iterator I = ++MBB->getIterator(); 11687 11688 Register DstReg = MI.getOperand(0).getReg(); 11689 const TargetRegisterClass *RC = MRI.getRegClass(DstReg); 11690 assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!"); 11691 Register mainDstReg = MRI.createVirtualRegister(RC); 11692 Register restoreDstReg = MRI.createVirtualRegister(RC); 11693 11694 MVT PVT = getPointerTy(MF->getDataLayout()); 11695 assert((PVT == MVT::i64 || PVT == MVT::i32) && 11696 "Invalid Pointer Size!"); 11697 // For v = setjmp(buf), we generate 11698 // 11699 // thisMBB: 11700 // SjLjSetup mainMBB 11701 // bl mainMBB 11702 // v_restore = 1 11703 // b sinkMBB 11704 // 11705 // mainMBB: 11706 // buf[LabelOffset] = LR 11707 // v_main = 0 11708 // 11709 // sinkMBB: 11710 // v = phi(main, restore) 11711 // 11712 11713 MachineBasicBlock *thisMBB = MBB; 11714 MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB); 11715 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB); 11716 MF->insert(I, mainMBB); 11717 MF->insert(I, sinkMBB); 11718 11719 MachineInstrBuilder MIB; 11720 11721 // Transfer the remainder of BB and its successor edges to sinkMBB. 11722 sinkMBB->splice(sinkMBB->begin(), MBB, 11723 std::next(MachineBasicBlock::iterator(MI)), MBB->end()); 11724 sinkMBB->transferSuccessorsAndUpdatePHIs(MBB); 11725 11726 // Note that the structure of the jmp_buf used here is not compatible 11727 // with that used by libc, and is not designed to be. Specifically, it 11728 // stores only those 'reserved' registers that LLVM does not otherwise 11729 // understand how to spill. Also, by convention, by the time this 11730 // intrinsic is called, Clang has already stored the frame address in the 11731 // first slot of the buffer and stack address in the third. Following the 11732 // X86 target code, we'll store the jump address in the second slot. We also 11733 // need to save the TOC pointer (R2) to handle jumps between shared 11734 // libraries, and that will be stored in the fourth slot. The thread 11735 // identifier (R13) is not affected. 11736 11737 // thisMBB: 11738 const int64_t LabelOffset = 1 * PVT.getStoreSize(); 11739 const int64_t TOCOffset = 3 * PVT.getStoreSize(); 11740 const int64_t BPOffset = 4 * PVT.getStoreSize(); 11741 11742 // Prepare IP either in reg. 11743 const TargetRegisterClass *PtrRC = getRegClassFor(PVT); 11744 Register LabelReg = MRI.createVirtualRegister(PtrRC); 11745 Register BufReg = MI.getOperand(1).getReg(); 11746 11747 if (Subtarget.is64BitELFABI()) { 11748 setUsesTOCBasePtr(*MBB->getParent()); 11749 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::STD)) 11750 .addReg(PPC::X2) 11751 .addImm(TOCOffset) 11752 .addReg(BufReg) 11753 .cloneMemRefs(MI); 11754 } 11755 11756 // Naked functions never have a base pointer, and so we use r1. For all 11757 // other functions, this decision must be delayed until during PEI. 11758 unsigned BaseReg; 11759 if (MF->getFunction().hasFnAttribute(Attribute::Naked)) 11760 BaseReg = Subtarget.isPPC64() ? PPC::X1 : PPC::R1; 11761 else 11762 BaseReg = Subtarget.isPPC64() ? PPC::BP8 : PPC::BP; 11763 11764 MIB = BuildMI(*thisMBB, MI, DL, 11765 TII->get(Subtarget.isPPC64() ? PPC::STD : PPC::STW)) 11766 .addReg(BaseReg) 11767 .addImm(BPOffset) 11768 .addReg(BufReg) 11769 .cloneMemRefs(MI); 11770 11771 // Setup 11772 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::BCLalways)).addMBB(mainMBB); 11773 MIB.addRegMask(TRI->getNoPreservedMask()); 11774 11775 BuildMI(*thisMBB, MI, DL, TII->get(PPC::LI), restoreDstReg).addImm(1); 11776 11777 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::EH_SjLj_Setup)) 11778 .addMBB(mainMBB); 11779 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::B)).addMBB(sinkMBB); 11780 11781 thisMBB->addSuccessor(mainMBB, BranchProbability::getZero()); 11782 thisMBB->addSuccessor(sinkMBB, BranchProbability::getOne()); 11783 11784 // mainMBB: 11785 // mainDstReg = 0 11786 MIB = 11787 BuildMI(mainMBB, DL, 11788 TII->get(Subtarget.isPPC64() ? PPC::MFLR8 : PPC::MFLR), LabelReg); 11789 11790 // Store IP 11791 if (Subtarget.isPPC64()) { 11792 MIB = BuildMI(mainMBB, DL, TII->get(PPC::STD)) 11793 .addReg(LabelReg) 11794 .addImm(LabelOffset) 11795 .addReg(BufReg); 11796 } else { 11797 MIB = BuildMI(mainMBB, DL, TII->get(PPC::STW)) 11798 .addReg(LabelReg) 11799 .addImm(LabelOffset) 11800 .addReg(BufReg); 11801 } 11802 MIB.cloneMemRefs(MI); 11803 11804 BuildMI(mainMBB, DL, TII->get(PPC::LI), mainDstReg).addImm(0); 11805 mainMBB->addSuccessor(sinkMBB); 11806 11807 // sinkMBB: 11808 BuildMI(*sinkMBB, sinkMBB->begin(), DL, 11809 TII->get(PPC::PHI), DstReg) 11810 .addReg(mainDstReg).addMBB(mainMBB) 11811 .addReg(restoreDstReg).addMBB(thisMBB); 11812 11813 MI.eraseFromParent(); 11814 return sinkMBB; 11815 } 11816 11817 MachineBasicBlock * 11818 PPCTargetLowering::emitEHSjLjLongJmp(MachineInstr &MI, 11819 MachineBasicBlock *MBB) const { 11820 DebugLoc DL = MI.getDebugLoc(); 11821 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 11822 11823 MachineFunction *MF = MBB->getParent(); 11824 MachineRegisterInfo &MRI = MF->getRegInfo(); 11825 11826 MVT PVT = getPointerTy(MF->getDataLayout()); 11827 assert((PVT == MVT::i64 || PVT == MVT::i32) && 11828 "Invalid Pointer Size!"); 11829 11830 const TargetRegisterClass *RC = 11831 (PVT == MVT::i64) ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; 11832 Register Tmp = MRI.createVirtualRegister(RC); 11833 // Since FP is only updated here but NOT referenced, it's treated as GPR. 11834 unsigned FP = (PVT == MVT::i64) ? PPC::X31 : PPC::R31; 11835 unsigned SP = (PVT == MVT::i64) ? PPC::X1 : PPC::R1; 11836 unsigned BP = 11837 (PVT == MVT::i64) 11838 ? PPC::X30 11839 : (Subtarget.isSVR4ABI() && isPositionIndependent() ? PPC::R29 11840 : PPC::R30); 11841 11842 MachineInstrBuilder MIB; 11843 11844 const int64_t LabelOffset = 1 * PVT.getStoreSize(); 11845 const int64_t SPOffset = 2 * PVT.getStoreSize(); 11846 const int64_t TOCOffset = 3 * PVT.getStoreSize(); 11847 const int64_t BPOffset = 4 * PVT.getStoreSize(); 11848 11849 Register BufReg = MI.getOperand(0).getReg(); 11850 11851 // Reload FP (the jumped-to function may not have had a 11852 // frame pointer, and if so, then its r31 will be restored 11853 // as necessary). 11854 if (PVT == MVT::i64) { 11855 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), FP) 11856 .addImm(0) 11857 .addReg(BufReg); 11858 } else { 11859 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), FP) 11860 .addImm(0) 11861 .addReg(BufReg); 11862 } 11863 MIB.cloneMemRefs(MI); 11864 11865 // Reload IP 11866 if (PVT == MVT::i64) { 11867 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), Tmp) 11868 .addImm(LabelOffset) 11869 .addReg(BufReg); 11870 } else { 11871 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), Tmp) 11872 .addImm(LabelOffset) 11873 .addReg(BufReg); 11874 } 11875 MIB.cloneMemRefs(MI); 11876 11877 // Reload SP 11878 if (PVT == MVT::i64) { 11879 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), SP) 11880 .addImm(SPOffset) 11881 .addReg(BufReg); 11882 } else { 11883 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), SP) 11884 .addImm(SPOffset) 11885 .addReg(BufReg); 11886 } 11887 MIB.cloneMemRefs(MI); 11888 11889 // Reload BP 11890 if (PVT == MVT::i64) { 11891 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), BP) 11892 .addImm(BPOffset) 11893 .addReg(BufReg); 11894 } else { 11895 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), BP) 11896 .addImm(BPOffset) 11897 .addReg(BufReg); 11898 } 11899 MIB.cloneMemRefs(MI); 11900 11901 // Reload TOC 11902 if (PVT == MVT::i64 && Subtarget.isSVR4ABI()) { 11903 setUsesTOCBasePtr(*MBB->getParent()); 11904 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), PPC::X2) 11905 .addImm(TOCOffset) 11906 .addReg(BufReg) 11907 .cloneMemRefs(MI); 11908 } 11909 11910 // Jump 11911 BuildMI(*MBB, MI, DL, 11912 TII->get(PVT == MVT::i64 ? PPC::MTCTR8 : PPC::MTCTR)).addReg(Tmp); 11913 BuildMI(*MBB, MI, DL, TII->get(PVT == MVT::i64 ? PPC::BCTR8 : PPC::BCTR)); 11914 11915 MI.eraseFromParent(); 11916 return MBB; 11917 } 11918 11919 bool PPCTargetLowering::hasInlineStackProbe(MachineFunction &MF) const { 11920 // If the function specifically requests inline stack probes, emit them. 11921 if (MF.getFunction().hasFnAttribute("probe-stack")) 11922 return MF.getFunction().getFnAttribute("probe-stack").getValueAsString() == 11923 "inline-asm"; 11924 return false; 11925 } 11926 11927 unsigned PPCTargetLowering::getStackProbeSize(MachineFunction &MF) const { 11928 const TargetFrameLowering *TFI = Subtarget.getFrameLowering(); 11929 unsigned StackAlign = TFI->getStackAlignment(); 11930 assert(StackAlign >= 1 && isPowerOf2_32(StackAlign) && 11931 "Unexpected stack alignment"); 11932 // The default stack probe size is 4096 if the function has no 11933 // stack-probe-size attribute. 11934 unsigned StackProbeSize = 4096; 11935 const Function &Fn = MF.getFunction(); 11936 if (Fn.hasFnAttribute("stack-probe-size")) 11937 Fn.getFnAttribute("stack-probe-size") 11938 .getValueAsString() 11939 .getAsInteger(0, StackProbeSize); 11940 // Round down to the stack alignment. 11941 StackProbeSize &= ~(StackAlign - 1); 11942 return StackProbeSize ? StackProbeSize : StackAlign; 11943 } 11944 11945 // Lower dynamic stack allocation with probing. `emitProbedAlloca` is splitted 11946 // into three phases. In the first phase, it uses pseudo instruction 11947 // PREPARE_PROBED_ALLOCA to get the future result of actual FramePointer and 11948 // FinalStackPtr. In the second phase, it generates a loop for probing blocks. 11949 // At last, it uses pseudo instruction DYNAREAOFFSET to get the future result of 11950 // MaxCallFrameSize so that it can calculate correct data area pointer. 11951 MachineBasicBlock * 11952 PPCTargetLowering::emitProbedAlloca(MachineInstr &MI, 11953 MachineBasicBlock *MBB) const { 11954 const bool isPPC64 = Subtarget.isPPC64(); 11955 MachineFunction *MF = MBB->getParent(); 11956 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 11957 DebugLoc DL = MI.getDebugLoc(); 11958 const unsigned ProbeSize = getStackProbeSize(*MF); 11959 const BasicBlock *ProbedBB = MBB->getBasicBlock(); 11960 MachineRegisterInfo &MRI = MF->getRegInfo(); 11961 // The CFG of probing stack looks as 11962 // +-----+ 11963 // | MBB | 11964 // +--+--+ 11965 // | 11966 // +----v----+ 11967 // +--->+ TestMBB +---+ 11968 // | +----+----+ | 11969 // | | | 11970 // | +-----v----+ | 11971 // +---+ BlockMBB | | 11972 // +----------+ | 11973 // | 11974 // +---------+ | 11975 // | TailMBB +<--+ 11976 // +---------+ 11977 // In MBB, calculate previous frame pointer and final stack pointer. 11978 // In TestMBB, test if sp is equal to final stack pointer, if so, jump to 11979 // TailMBB. In BlockMBB, update the sp atomically and jump back to TestMBB. 11980 // TailMBB is spliced via \p MI. 11981 MachineBasicBlock *TestMBB = MF->CreateMachineBasicBlock(ProbedBB); 11982 MachineBasicBlock *TailMBB = MF->CreateMachineBasicBlock(ProbedBB); 11983 MachineBasicBlock *BlockMBB = MF->CreateMachineBasicBlock(ProbedBB); 11984 11985 MachineFunction::iterator MBBIter = ++MBB->getIterator(); 11986 MF->insert(MBBIter, TestMBB); 11987 MF->insert(MBBIter, BlockMBB); 11988 MF->insert(MBBIter, TailMBB); 11989 11990 const TargetRegisterClass *G8RC = &PPC::G8RCRegClass; 11991 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass; 11992 11993 Register DstReg = MI.getOperand(0).getReg(); 11994 Register NegSizeReg = MI.getOperand(1).getReg(); 11995 Register SPReg = isPPC64 ? PPC::X1 : PPC::R1; 11996 Register FinalStackPtr = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC); 11997 Register FramePointer = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC); 11998 Register ActualNegSizeReg = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC); 11999 12000 // Since value of NegSizeReg might be realigned in prologepilog, insert a 12001 // PREPARE_PROBED_ALLOCA pseudo instruction to get actual FramePointer and 12002 // NegSize. 12003 unsigned ProbeOpc; 12004 if (!MRI.hasOneNonDBGUse(NegSizeReg)) 12005 ProbeOpc = 12006 isPPC64 ? PPC::PREPARE_PROBED_ALLOCA_64 : PPC::PREPARE_PROBED_ALLOCA_32; 12007 else 12008 // By introducing PREPARE_PROBED_ALLOCA_NEGSIZE_OPT, ActualNegSizeReg 12009 // and NegSizeReg will be allocated in the same phyreg to avoid 12010 // redundant copy when NegSizeReg has only one use which is current MI and 12011 // will be replaced by PREPARE_PROBED_ALLOCA then. 12012 ProbeOpc = isPPC64 ? PPC::PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_64 12013 : PPC::PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_32; 12014 BuildMI(*MBB, {MI}, DL, TII->get(ProbeOpc), FramePointer) 12015 .addDef(ActualNegSizeReg) 12016 .addReg(NegSizeReg) 12017 .add(MI.getOperand(2)) 12018 .add(MI.getOperand(3)); 12019 12020 // Calculate final stack pointer, which equals to SP + ActualNegSize. 12021 BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::ADD8 : PPC::ADD4), 12022 FinalStackPtr) 12023 .addReg(SPReg) 12024 .addReg(ActualNegSizeReg); 12025 12026 // Materialize a scratch register for update. 12027 int64_t NegProbeSize = -(int64_t)ProbeSize; 12028 assert(isInt<32>(NegProbeSize) && "Unhandled probe size!"); 12029 Register ScratchReg = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC); 12030 if (!isInt<16>(NegProbeSize)) { 12031 Register TempReg = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC); 12032 BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::LIS8 : PPC::LIS), TempReg) 12033 .addImm(NegProbeSize >> 16); 12034 BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::ORI8 : PPC::ORI), 12035 ScratchReg) 12036 .addReg(TempReg) 12037 .addImm(NegProbeSize & 0xFFFF); 12038 } else 12039 BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::LI8 : PPC::LI), ScratchReg) 12040 .addImm(NegProbeSize); 12041 12042 { 12043 // Probing leading residual part. 12044 Register Div = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC); 12045 BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::DIVD : PPC::DIVW), Div) 12046 .addReg(ActualNegSizeReg) 12047 .addReg(ScratchReg); 12048 Register Mul = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC); 12049 BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::MULLD : PPC::MULLW), Mul) 12050 .addReg(Div) 12051 .addReg(ScratchReg); 12052 Register NegMod = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC); 12053 BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::SUBF8 : PPC::SUBF), NegMod) 12054 .addReg(Mul) 12055 .addReg(ActualNegSizeReg); 12056 BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::STDUX : PPC::STWUX), SPReg) 12057 .addReg(FramePointer) 12058 .addReg(SPReg) 12059 .addReg(NegMod); 12060 } 12061 12062 { 12063 // Remaining part should be multiple of ProbeSize. 12064 Register CmpResult = MRI.createVirtualRegister(&PPC::CRRCRegClass); 12065 BuildMI(TestMBB, DL, TII->get(isPPC64 ? PPC::CMPD : PPC::CMPW), CmpResult) 12066 .addReg(SPReg) 12067 .addReg(FinalStackPtr); 12068 BuildMI(TestMBB, DL, TII->get(PPC::BCC)) 12069 .addImm(PPC::PRED_EQ) 12070 .addReg(CmpResult) 12071 .addMBB(TailMBB); 12072 TestMBB->addSuccessor(BlockMBB); 12073 TestMBB->addSuccessor(TailMBB); 12074 } 12075 12076 { 12077 // Touch the block. 12078 // |P...|P...|P... 12079 BuildMI(BlockMBB, DL, TII->get(isPPC64 ? PPC::STDUX : PPC::STWUX), SPReg) 12080 .addReg(FramePointer) 12081 .addReg(SPReg) 12082 .addReg(ScratchReg); 12083 BuildMI(BlockMBB, DL, TII->get(PPC::B)).addMBB(TestMBB); 12084 BlockMBB->addSuccessor(TestMBB); 12085 } 12086 12087 // Calculation of MaxCallFrameSize is deferred to prologepilog, use 12088 // DYNAREAOFFSET pseudo instruction to get the future result. 12089 Register MaxCallFrameSizeReg = 12090 MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC); 12091 BuildMI(TailMBB, DL, 12092 TII->get(isPPC64 ? PPC::DYNAREAOFFSET8 : PPC::DYNAREAOFFSET), 12093 MaxCallFrameSizeReg) 12094 .add(MI.getOperand(2)) 12095 .add(MI.getOperand(3)); 12096 BuildMI(TailMBB, DL, TII->get(isPPC64 ? PPC::ADD8 : PPC::ADD4), DstReg) 12097 .addReg(SPReg) 12098 .addReg(MaxCallFrameSizeReg); 12099 12100 // Splice instructions after MI to TailMBB. 12101 TailMBB->splice(TailMBB->end(), MBB, 12102 std::next(MachineBasicBlock::iterator(MI)), MBB->end()); 12103 TailMBB->transferSuccessorsAndUpdatePHIs(MBB); 12104 MBB->addSuccessor(TestMBB); 12105 12106 // Delete the pseudo instruction. 12107 MI.eraseFromParent(); 12108 12109 ++NumDynamicAllocaProbed; 12110 return TailMBB; 12111 } 12112 12113 MachineBasicBlock * 12114 PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, 12115 MachineBasicBlock *BB) const { 12116 if (MI.getOpcode() == TargetOpcode::STACKMAP || 12117 MI.getOpcode() == TargetOpcode::PATCHPOINT) { 12118 if (Subtarget.is64BitELFABI() && 12119 MI.getOpcode() == TargetOpcode::PATCHPOINT && 12120 !Subtarget.isUsingPCRelativeCalls()) { 12121 // Call lowering should have added an r2 operand to indicate a dependence 12122 // on the TOC base pointer value. It can't however, because there is no 12123 // way to mark the dependence as implicit there, and so the stackmap code 12124 // will confuse it with a regular operand. Instead, add the dependence 12125 // here. 12126 MI.addOperand(MachineOperand::CreateReg(PPC::X2, false, true)); 12127 } 12128 12129 return emitPatchPoint(MI, BB); 12130 } 12131 12132 if (MI.getOpcode() == PPC::EH_SjLj_SetJmp32 || 12133 MI.getOpcode() == PPC::EH_SjLj_SetJmp64) { 12134 return emitEHSjLjSetJmp(MI, BB); 12135 } else if (MI.getOpcode() == PPC::EH_SjLj_LongJmp32 || 12136 MI.getOpcode() == PPC::EH_SjLj_LongJmp64) { 12137 return emitEHSjLjLongJmp(MI, BB); 12138 } 12139 12140 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 12141 12142 // To "insert" these instructions we actually have to insert their 12143 // control-flow patterns. 12144 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 12145 MachineFunction::iterator It = ++BB->getIterator(); 12146 12147 MachineFunction *F = BB->getParent(); 12148 MachineRegisterInfo &MRI = F->getRegInfo(); 12149 12150 if (MI.getOpcode() == PPC::SELECT_CC_I4 || 12151 MI.getOpcode() == PPC::SELECT_CC_I8 || MI.getOpcode() == PPC::SELECT_I4 || 12152 MI.getOpcode() == PPC::SELECT_I8) { 12153 SmallVector<MachineOperand, 2> Cond; 12154 if (MI.getOpcode() == PPC::SELECT_CC_I4 || 12155 MI.getOpcode() == PPC::SELECT_CC_I8) 12156 Cond.push_back(MI.getOperand(4)); 12157 else 12158 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET)); 12159 Cond.push_back(MI.getOperand(1)); 12160 12161 DebugLoc dl = MI.getDebugLoc(); 12162 TII->insertSelect(*BB, MI, dl, MI.getOperand(0).getReg(), Cond, 12163 MI.getOperand(2).getReg(), MI.getOperand(3).getReg()); 12164 } else if (MI.getOpcode() == PPC::SELECT_CC_F4 || 12165 MI.getOpcode() == PPC::SELECT_CC_F8 || 12166 MI.getOpcode() == PPC::SELECT_CC_F16 || 12167 MI.getOpcode() == PPC::SELECT_CC_VRRC || 12168 MI.getOpcode() == PPC::SELECT_CC_VSFRC || 12169 MI.getOpcode() == PPC::SELECT_CC_VSSRC || 12170 MI.getOpcode() == PPC::SELECT_CC_VSRC || 12171 MI.getOpcode() == PPC::SELECT_CC_SPE4 || 12172 MI.getOpcode() == PPC::SELECT_CC_SPE || 12173 MI.getOpcode() == PPC::SELECT_F4 || 12174 MI.getOpcode() == PPC::SELECT_F8 || 12175 MI.getOpcode() == PPC::SELECT_F16 || 12176 MI.getOpcode() == PPC::SELECT_SPE || 12177 MI.getOpcode() == PPC::SELECT_SPE4 || 12178 MI.getOpcode() == PPC::SELECT_VRRC || 12179 MI.getOpcode() == PPC::SELECT_VSFRC || 12180 MI.getOpcode() == PPC::SELECT_VSSRC || 12181 MI.getOpcode() == PPC::SELECT_VSRC) { 12182 // The incoming instruction knows the destination vreg to set, the 12183 // condition code register to branch on, the true/false values to 12184 // select between, and a branch opcode to use. 12185 12186 // thisMBB: 12187 // ... 12188 // TrueVal = ... 12189 // cmpTY ccX, r1, r2 12190 // bCC copy1MBB 12191 // fallthrough --> copy0MBB 12192 MachineBasicBlock *thisMBB = BB; 12193 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); 12194 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); 12195 DebugLoc dl = MI.getDebugLoc(); 12196 F->insert(It, copy0MBB); 12197 F->insert(It, sinkMBB); 12198 12199 // Transfer the remainder of BB and its successor edges to sinkMBB. 12200 sinkMBB->splice(sinkMBB->begin(), BB, 12201 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 12202 sinkMBB->transferSuccessorsAndUpdatePHIs(BB); 12203 12204 // Next, add the true and fallthrough blocks as its successors. 12205 BB->addSuccessor(copy0MBB); 12206 BB->addSuccessor(sinkMBB); 12207 12208 if (MI.getOpcode() == PPC::SELECT_I4 || MI.getOpcode() == PPC::SELECT_I8 || 12209 MI.getOpcode() == PPC::SELECT_F4 || MI.getOpcode() == PPC::SELECT_F8 || 12210 MI.getOpcode() == PPC::SELECT_F16 || 12211 MI.getOpcode() == PPC::SELECT_SPE4 || 12212 MI.getOpcode() == PPC::SELECT_SPE || 12213 MI.getOpcode() == PPC::SELECT_VRRC || 12214 MI.getOpcode() == PPC::SELECT_VSFRC || 12215 MI.getOpcode() == PPC::SELECT_VSSRC || 12216 MI.getOpcode() == PPC::SELECT_VSRC) { 12217 BuildMI(BB, dl, TII->get(PPC::BC)) 12218 .addReg(MI.getOperand(1).getReg()) 12219 .addMBB(sinkMBB); 12220 } else { 12221 unsigned SelectPred = MI.getOperand(4).getImm(); 12222 BuildMI(BB, dl, TII->get(PPC::BCC)) 12223 .addImm(SelectPred) 12224 .addReg(MI.getOperand(1).getReg()) 12225 .addMBB(sinkMBB); 12226 } 12227 12228 // copy0MBB: 12229 // %FalseValue = ... 12230 // # fallthrough to sinkMBB 12231 BB = copy0MBB; 12232 12233 // Update machine-CFG edges 12234 BB->addSuccessor(sinkMBB); 12235 12236 // sinkMBB: 12237 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] 12238 // ... 12239 BB = sinkMBB; 12240 BuildMI(*BB, BB->begin(), dl, TII->get(PPC::PHI), MI.getOperand(0).getReg()) 12241 .addReg(MI.getOperand(3).getReg()) 12242 .addMBB(copy0MBB) 12243 .addReg(MI.getOperand(2).getReg()) 12244 .addMBB(thisMBB); 12245 } else if (MI.getOpcode() == PPC::ReadTB) { 12246 // To read the 64-bit time-base register on a 32-bit target, we read the 12247 // two halves. Should the counter have wrapped while it was being read, we 12248 // need to try again. 12249 // ... 12250 // readLoop: 12251 // mfspr Rx,TBU # load from TBU 12252 // mfspr Ry,TB # load from TB 12253 // mfspr Rz,TBU # load from TBU 12254 // cmpw crX,Rx,Rz # check if 'old'='new' 12255 // bne readLoop # branch if they're not equal 12256 // ... 12257 12258 MachineBasicBlock *readMBB = F->CreateMachineBasicBlock(LLVM_BB); 12259 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); 12260 DebugLoc dl = MI.getDebugLoc(); 12261 F->insert(It, readMBB); 12262 F->insert(It, sinkMBB); 12263 12264 // Transfer the remainder of BB and its successor edges to sinkMBB. 12265 sinkMBB->splice(sinkMBB->begin(), BB, 12266 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 12267 sinkMBB->transferSuccessorsAndUpdatePHIs(BB); 12268 12269 BB->addSuccessor(readMBB); 12270 BB = readMBB; 12271 12272 MachineRegisterInfo &RegInfo = F->getRegInfo(); 12273 Register ReadAgainReg = RegInfo.createVirtualRegister(&PPC::GPRCRegClass); 12274 Register LoReg = MI.getOperand(0).getReg(); 12275 Register HiReg = MI.getOperand(1).getReg(); 12276 12277 BuildMI(BB, dl, TII->get(PPC::MFSPR), HiReg).addImm(269); 12278 BuildMI(BB, dl, TII->get(PPC::MFSPR), LoReg).addImm(268); 12279 BuildMI(BB, dl, TII->get(PPC::MFSPR), ReadAgainReg).addImm(269); 12280 12281 Register CmpReg = RegInfo.createVirtualRegister(&PPC::CRRCRegClass); 12282 12283 BuildMI(BB, dl, TII->get(PPC::CMPW), CmpReg) 12284 .addReg(HiReg) 12285 .addReg(ReadAgainReg); 12286 BuildMI(BB, dl, TII->get(PPC::BCC)) 12287 .addImm(PPC::PRED_NE) 12288 .addReg(CmpReg) 12289 .addMBB(readMBB); 12290 12291 BB->addSuccessor(readMBB); 12292 BB->addSuccessor(sinkMBB); 12293 } else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I8) 12294 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::ADD4); 12295 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I16) 12296 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::ADD4); 12297 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I32) 12298 BB = EmitAtomicBinary(MI, BB, 4, PPC::ADD4); 12299 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I64) 12300 BB = EmitAtomicBinary(MI, BB, 8, PPC::ADD8); 12301 12302 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I8) 12303 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::AND); 12304 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I16) 12305 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::AND); 12306 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I32) 12307 BB = EmitAtomicBinary(MI, BB, 4, PPC::AND); 12308 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I64) 12309 BB = EmitAtomicBinary(MI, BB, 8, PPC::AND8); 12310 12311 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I8) 12312 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::OR); 12313 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I16) 12314 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::OR); 12315 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I32) 12316 BB = EmitAtomicBinary(MI, BB, 4, PPC::OR); 12317 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I64) 12318 BB = EmitAtomicBinary(MI, BB, 8, PPC::OR8); 12319 12320 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I8) 12321 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::XOR); 12322 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I16) 12323 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::XOR); 12324 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I32) 12325 BB = EmitAtomicBinary(MI, BB, 4, PPC::XOR); 12326 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I64) 12327 BB = EmitAtomicBinary(MI, BB, 8, PPC::XOR8); 12328 12329 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I8) 12330 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::NAND); 12331 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I16) 12332 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::NAND); 12333 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I32) 12334 BB = EmitAtomicBinary(MI, BB, 4, PPC::NAND); 12335 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I64) 12336 BB = EmitAtomicBinary(MI, BB, 8, PPC::NAND8); 12337 12338 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I8) 12339 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::SUBF); 12340 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I16) 12341 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::SUBF); 12342 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I32) 12343 BB = EmitAtomicBinary(MI, BB, 4, PPC::SUBF); 12344 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I64) 12345 BB = EmitAtomicBinary(MI, BB, 8, PPC::SUBF8); 12346 12347 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I8) 12348 BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPW, PPC::PRED_GE); 12349 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I16) 12350 BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPW, PPC::PRED_GE); 12351 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I32) 12352 BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPW, PPC::PRED_GE); 12353 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I64) 12354 BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPD, PPC::PRED_GE); 12355 12356 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I8) 12357 BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPW, PPC::PRED_LE); 12358 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I16) 12359 BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPW, PPC::PRED_LE); 12360 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I32) 12361 BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPW, PPC::PRED_LE); 12362 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I64) 12363 BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPD, PPC::PRED_LE); 12364 12365 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I8) 12366 BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPLW, PPC::PRED_GE); 12367 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I16) 12368 BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPLW, PPC::PRED_GE); 12369 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I32) 12370 BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPLW, PPC::PRED_GE); 12371 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I64) 12372 BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPLD, PPC::PRED_GE); 12373 12374 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I8) 12375 BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPLW, PPC::PRED_LE); 12376 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I16) 12377 BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPLW, PPC::PRED_LE); 12378 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I32) 12379 BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPLW, PPC::PRED_LE); 12380 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I64) 12381 BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPLD, PPC::PRED_LE); 12382 12383 else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I8) 12384 BB = EmitPartwordAtomicBinary(MI, BB, true, 0); 12385 else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I16) 12386 BB = EmitPartwordAtomicBinary(MI, BB, false, 0); 12387 else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I32) 12388 BB = EmitAtomicBinary(MI, BB, 4, 0); 12389 else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I64) 12390 BB = EmitAtomicBinary(MI, BB, 8, 0); 12391 else if (MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I32 || 12392 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I64 || 12393 (Subtarget.hasPartwordAtomics() && 12394 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8) || 12395 (Subtarget.hasPartwordAtomics() && 12396 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I16)) { 12397 bool is64bit = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I64; 12398 12399 auto LoadMnemonic = PPC::LDARX; 12400 auto StoreMnemonic = PPC::STDCX; 12401 switch (MI.getOpcode()) { 12402 default: 12403 llvm_unreachable("Compare and swap of unknown size"); 12404 case PPC::ATOMIC_CMP_SWAP_I8: 12405 LoadMnemonic = PPC::LBARX; 12406 StoreMnemonic = PPC::STBCX; 12407 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics."); 12408 break; 12409 case PPC::ATOMIC_CMP_SWAP_I16: 12410 LoadMnemonic = PPC::LHARX; 12411 StoreMnemonic = PPC::STHCX; 12412 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics."); 12413 break; 12414 case PPC::ATOMIC_CMP_SWAP_I32: 12415 LoadMnemonic = PPC::LWARX; 12416 StoreMnemonic = PPC::STWCX; 12417 break; 12418 case PPC::ATOMIC_CMP_SWAP_I64: 12419 LoadMnemonic = PPC::LDARX; 12420 StoreMnemonic = PPC::STDCX; 12421 break; 12422 } 12423 Register dest = MI.getOperand(0).getReg(); 12424 Register ptrA = MI.getOperand(1).getReg(); 12425 Register ptrB = MI.getOperand(2).getReg(); 12426 Register oldval = MI.getOperand(3).getReg(); 12427 Register newval = MI.getOperand(4).getReg(); 12428 DebugLoc dl = MI.getDebugLoc(); 12429 12430 MachineBasicBlock *loop1MBB = F->CreateMachineBasicBlock(LLVM_BB); 12431 MachineBasicBlock *loop2MBB = F->CreateMachineBasicBlock(LLVM_BB); 12432 MachineBasicBlock *midMBB = F->CreateMachineBasicBlock(LLVM_BB); 12433 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); 12434 F->insert(It, loop1MBB); 12435 F->insert(It, loop2MBB); 12436 F->insert(It, midMBB); 12437 F->insert(It, exitMBB); 12438 exitMBB->splice(exitMBB->begin(), BB, 12439 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 12440 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 12441 12442 // thisMBB: 12443 // ... 12444 // fallthrough --> loopMBB 12445 BB->addSuccessor(loop1MBB); 12446 12447 // loop1MBB: 12448 // l[bhwd]arx dest, ptr 12449 // cmp[wd] dest, oldval 12450 // bne- midMBB 12451 // loop2MBB: 12452 // st[bhwd]cx. newval, ptr 12453 // bne- loopMBB 12454 // b exitBB 12455 // midMBB: 12456 // st[bhwd]cx. dest, ptr 12457 // exitBB: 12458 BB = loop1MBB; 12459 BuildMI(BB, dl, TII->get(LoadMnemonic), dest).addReg(ptrA).addReg(ptrB); 12460 BuildMI(BB, dl, TII->get(is64bit ? PPC::CMPD : PPC::CMPW), PPC::CR0) 12461 .addReg(oldval) 12462 .addReg(dest); 12463 BuildMI(BB, dl, TII->get(PPC::BCC)) 12464 .addImm(PPC::PRED_NE) 12465 .addReg(PPC::CR0) 12466 .addMBB(midMBB); 12467 BB->addSuccessor(loop2MBB); 12468 BB->addSuccessor(midMBB); 12469 12470 BB = loop2MBB; 12471 BuildMI(BB, dl, TII->get(StoreMnemonic)) 12472 .addReg(newval) 12473 .addReg(ptrA) 12474 .addReg(ptrB); 12475 BuildMI(BB, dl, TII->get(PPC::BCC)) 12476 .addImm(PPC::PRED_NE) 12477 .addReg(PPC::CR0) 12478 .addMBB(loop1MBB); 12479 BuildMI(BB, dl, TII->get(PPC::B)).addMBB(exitMBB); 12480 BB->addSuccessor(loop1MBB); 12481 BB->addSuccessor(exitMBB); 12482 12483 BB = midMBB; 12484 BuildMI(BB, dl, TII->get(StoreMnemonic)) 12485 .addReg(dest) 12486 .addReg(ptrA) 12487 .addReg(ptrB); 12488 BB->addSuccessor(exitMBB); 12489 12490 // exitMBB: 12491 // ... 12492 BB = exitMBB; 12493 } else if (MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8 || 12494 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I16) { 12495 // We must use 64-bit registers for addresses when targeting 64-bit, 12496 // since we're actually doing arithmetic on them. Other registers 12497 // can be 32-bit. 12498 bool is64bit = Subtarget.isPPC64(); 12499 bool isLittleEndian = Subtarget.isLittleEndian(); 12500 bool is8bit = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8; 12501 12502 Register dest = MI.getOperand(0).getReg(); 12503 Register ptrA = MI.getOperand(1).getReg(); 12504 Register ptrB = MI.getOperand(2).getReg(); 12505 Register oldval = MI.getOperand(3).getReg(); 12506 Register newval = MI.getOperand(4).getReg(); 12507 DebugLoc dl = MI.getDebugLoc(); 12508 12509 MachineBasicBlock *loop1MBB = F->CreateMachineBasicBlock(LLVM_BB); 12510 MachineBasicBlock *loop2MBB = F->CreateMachineBasicBlock(LLVM_BB); 12511 MachineBasicBlock *midMBB = F->CreateMachineBasicBlock(LLVM_BB); 12512 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); 12513 F->insert(It, loop1MBB); 12514 F->insert(It, loop2MBB); 12515 F->insert(It, midMBB); 12516 F->insert(It, exitMBB); 12517 exitMBB->splice(exitMBB->begin(), BB, 12518 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 12519 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 12520 12521 MachineRegisterInfo &RegInfo = F->getRegInfo(); 12522 const TargetRegisterClass *RC = 12523 is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; 12524 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass; 12525 12526 Register PtrReg = RegInfo.createVirtualRegister(RC); 12527 Register Shift1Reg = RegInfo.createVirtualRegister(GPRC); 12528 Register ShiftReg = 12529 isLittleEndian ? Shift1Reg : RegInfo.createVirtualRegister(GPRC); 12530 Register NewVal2Reg = RegInfo.createVirtualRegister(GPRC); 12531 Register NewVal3Reg = RegInfo.createVirtualRegister(GPRC); 12532 Register OldVal2Reg = RegInfo.createVirtualRegister(GPRC); 12533 Register OldVal3Reg = RegInfo.createVirtualRegister(GPRC); 12534 Register MaskReg = RegInfo.createVirtualRegister(GPRC); 12535 Register Mask2Reg = RegInfo.createVirtualRegister(GPRC); 12536 Register Mask3Reg = RegInfo.createVirtualRegister(GPRC); 12537 Register Tmp2Reg = RegInfo.createVirtualRegister(GPRC); 12538 Register Tmp4Reg = RegInfo.createVirtualRegister(GPRC); 12539 Register TmpDestReg = RegInfo.createVirtualRegister(GPRC); 12540 Register Ptr1Reg; 12541 Register TmpReg = RegInfo.createVirtualRegister(GPRC); 12542 Register ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO; 12543 // thisMBB: 12544 // ... 12545 // fallthrough --> loopMBB 12546 BB->addSuccessor(loop1MBB); 12547 12548 // The 4-byte load must be aligned, while a char or short may be 12549 // anywhere in the word. Hence all this nasty bookkeeping code. 12550 // add ptr1, ptrA, ptrB [copy if ptrA==0] 12551 // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27] 12552 // xori shift, shift1, 24 [16] 12553 // rlwinm ptr, ptr1, 0, 0, 29 12554 // slw newval2, newval, shift 12555 // slw oldval2, oldval,shift 12556 // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535] 12557 // slw mask, mask2, shift 12558 // and newval3, newval2, mask 12559 // and oldval3, oldval2, mask 12560 // loop1MBB: 12561 // lwarx tmpDest, ptr 12562 // and tmp, tmpDest, mask 12563 // cmpw tmp, oldval3 12564 // bne- midMBB 12565 // loop2MBB: 12566 // andc tmp2, tmpDest, mask 12567 // or tmp4, tmp2, newval3 12568 // stwcx. tmp4, ptr 12569 // bne- loop1MBB 12570 // b exitBB 12571 // midMBB: 12572 // stwcx. tmpDest, ptr 12573 // exitBB: 12574 // srw dest, tmpDest, shift 12575 if (ptrA != ZeroReg) { 12576 Ptr1Reg = RegInfo.createVirtualRegister(RC); 12577 BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg) 12578 .addReg(ptrA) 12579 .addReg(ptrB); 12580 } else { 12581 Ptr1Reg = ptrB; 12582 } 12583 12584 // We need use 32-bit subregister to avoid mismatch register class in 64-bit 12585 // mode. 12586 BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg) 12587 .addReg(Ptr1Reg, 0, is64bit ? PPC::sub_32 : 0) 12588 .addImm(3) 12589 .addImm(27) 12590 .addImm(is8bit ? 28 : 27); 12591 if (!isLittleEndian) 12592 BuildMI(BB, dl, TII->get(PPC::XORI), ShiftReg) 12593 .addReg(Shift1Reg) 12594 .addImm(is8bit ? 24 : 16); 12595 if (is64bit) 12596 BuildMI(BB, dl, TII->get(PPC::RLDICR), PtrReg) 12597 .addReg(Ptr1Reg) 12598 .addImm(0) 12599 .addImm(61); 12600 else 12601 BuildMI(BB, dl, TII->get(PPC::RLWINM), PtrReg) 12602 .addReg(Ptr1Reg) 12603 .addImm(0) 12604 .addImm(0) 12605 .addImm(29); 12606 BuildMI(BB, dl, TII->get(PPC::SLW), NewVal2Reg) 12607 .addReg(newval) 12608 .addReg(ShiftReg); 12609 BuildMI(BB, dl, TII->get(PPC::SLW), OldVal2Reg) 12610 .addReg(oldval) 12611 .addReg(ShiftReg); 12612 if (is8bit) 12613 BuildMI(BB, dl, TII->get(PPC::LI), Mask2Reg).addImm(255); 12614 else { 12615 BuildMI(BB, dl, TII->get(PPC::LI), Mask3Reg).addImm(0); 12616 BuildMI(BB, dl, TII->get(PPC::ORI), Mask2Reg) 12617 .addReg(Mask3Reg) 12618 .addImm(65535); 12619 } 12620 BuildMI(BB, dl, TII->get(PPC::SLW), MaskReg) 12621 .addReg(Mask2Reg) 12622 .addReg(ShiftReg); 12623 BuildMI(BB, dl, TII->get(PPC::AND), NewVal3Reg) 12624 .addReg(NewVal2Reg) 12625 .addReg(MaskReg); 12626 BuildMI(BB, dl, TII->get(PPC::AND), OldVal3Reg) 12627 .addReg(OldVal2Reg) 12628 .addReg(MaskReg); 12629 12630 BB = loop1MBB; 12631 BuildMI(BB, dl, TII->get(PPC::LWARX), TmpDestReg) 12632 .addReg(ZeroReg) 12633 .addReg(PtrReg); 12634 BuildMI(BB, dl, TII->get(PPC::AND), TmpReg) 12635 .addReg(TmpDestReg) 12636 .addReg(MaskReg); 12637 BuildMI(BB, dl, TII->get(PPC::CMPW), PPC::CR0) 12638 .addReg(TmpReg) 12639 .addReg(OldVal3Reg); 12640 BuildMI(BB, dl, TII->get(PPC::BCC)) 12641 .addImm(PPC::PRED_NE) 12642 .addReg(PPC::CR0) 12643 .addMBB(midMBB); 12644 BB->addSuccessor(loop2MBB); 12645 BB->addSuccessor(midMBB); 12646 12647 BB = loop2MBB; 12648 BuildMI(BB, dl, TII->get(PPC::ANDC), Tmp2Reg) 12649 .addReg(TmpDestReg) 12650 .addReg(MaskReg); 12651 BuildMI(BB, dl, TII->get(PPC::OR), Tmp4Reg) 12652 .addReg(Tmp2Reg) 12653 .addReg(NewVal3Reg); 12654 BuildMI(BB, dl, TII->get(PPC::STWCX)) 12655 .addReg(Tmp4Reg) 12656 .addReg(ZeroReg) 12657 .addReg(PtrReg); 12658 BuildMI(BB, dl, TII->get(PPC::BCC)) 12659 .addImm(PPC::PRED_NE) 12660 .addReg(PPC::CR0) 12661 .addMBB(loop1MBB); 12662 BuildMI(BB, dl, TII->get(PPC::B)).addMBB(exitMBB); 12663 BB->addSuccessor(loop1MBB); 12664 BB->addSuccessor(exitMBB); 12665 12666 BB = midMBB; 12667 BuildMI(BB, dl, TII->get(PPC::STWCX)) 12668 .addReg(TmpDestReg) 12669 .addReg(ZeroReg) 12670 .addReg(PtrReg); 12671 BB->addSuccessor(exitMBB); 12672 12673 // exitMBB: 12674 // ... 12675 BB = exitMBB; 12676 BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), dest) 12677 .addReg(TmpReg) 12678 .addReg(ShiftReg); 12679 } else if (MI.getOpcode() == PPC::FADDrtz) { 12680 // This pseudo performs an FADD with rounding mode temporarily forced 12681 // to round-to-zero. We emit this via custom inserter since the FPSCR 12682 // is not modeled at the SelectionDAG level. 12683 Register Dest = MI.getOperand(0).getReg(); 12684 Register Src1 = MI.getOperand(1).getReg(); 12685 Register Src2 = MI.getOperand(2).getReg(); 12686 DebugLoc dl = MI.getDebugLoc(); 12687 12688 MachineRegisterInfo &RegInfo = F->getRegInfo(); 12689 Register MFFSReg = RegInfo.createVirtualRegister(&PPC::F8RCRegClass); 12690 12691 // Save FPSCR value. 12692 BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), MFFSReg); 12693 12694 // Set rounding mode to round-to-zero. 12695 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB1)) 12696 .addImm(31) 12697 .addReg(PPC::RM, RegState::ImplicitDefine); 12698 12699 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB0)) 12700 .addImm(30) 12701 .addReg(PPC::RM, RegState::ImplicitDefine); 12702 12703 // Perform addition. 12704 auto MIB = BuildMI(*BB, MI, dl, TII->get(PPC::FADD), Dest) 12705 .addReg(Src1) 12706 .addReg(Src2); 12707 if (MI.getFlag(MachineInstr::NoFPExcept)) 12708 MIB.setMIFlag(MachineInstr::NoFPExcept); 12709 12710 // Restore FPSCR value. 12711 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSFb)).addImm(1).addReg(MFFSReg); 12712 } else if (MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT || 12713 MI.getOpcode() == PPC::ANDI_rec_1_GT_BIT || 12714 MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT8 || 12715 MI.getOpcode() == PPC::ANDI_rec_1_GT_BIT8) { 12716 unsigned Opcode = (MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT8 || 12717 MI.getOpcode() == PPC::ANDI_rec_1_GT_BIT8) 12718 ? PPC::ANDI8_rec 12719 : PPC::ANDI_rec; 12720 bool IsEQ = (MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT || 12721 MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT8); 12722 12723 MachineRegisterInfo &RegInfo = F->getRegInfo(); 12724 Register Dest = RegInfo.createVirtualRegister( 12725 Opcode == PPC::ANDI_rec ? &PPC::GPRCRegClass : &PPC::G8RCRegClass); 12726 12727 DebugLoc Dl = MI.getDebugLoc(); 12728 BuildMI(*BB, MI, Dl, TII->get(Opcode), Dest) 12729 .addReg(MI.getOperand(1).getReg()) 12730 .addImm(1); 12731 BuildMI(*BB, MI, Dl, TII->get(TargetOpcode::COPY), 12732 MI.getOperand(0).getReg()) 12733 .addReg(IsEQ ? PPC::CR0EQ : PPC::CR0GT); 12734 } else if (MI.getOpcode() == PPC::TCHECK_RET) { 12735 DebugLoc Dl = MI.getDebugLoc(); 12736 MachineRegisterInfo &RegInfo = F->getRegInfo(); 12737 Register CRReg = RegInfo.createVirtualRegister(&PPC::CRRCRegClass); 12738 BuildMI(*BB, MI, Dl, TII->get(PPC::TCHECK), CRReg); 12739 BuildMI(*BB, MI, Dl, TII->get(TargetOpcode::COPY), 12740 MI.getOperand(0).getReg()) 12741 .addReg(CRReg); 12742 } else if (MI.getOpcode() == PPC::TBEGIN_RET) { 12743 DebugLoc Dl = MI.getDebugLoc(); 12744 unsigned Imm = MI.getOperand(1).getImm(); 12745 BuildMI(*BB, MI, Dl, TII->get(PPC::TBEGIN)).addImm(Imm); 12746 BuildMI(*BB, MI, Dl, TII->get(TargetOpcode::COPY), 12747 MI.getOperand(0).getReg()) 12748 .addReg(PPC::CR0EQ); 12749 } else if (MI.getOpcode() == PPC::SETRNDi) { 12750 DebugLoc dl = MI.getDebugLoc(); 12751 Register OldFPSCRReg = MI.getOperand(0).getReg(); 12752 12753 // Save FPSCR value. 12754 if (MRI.use_empty(OldFPSCRReg)) 12755 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::IMPLICIT_DEF), OldFPSCRReg); 12756 else 12757 BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), OldFPSCRReg); 12758 12759 // The floating point rounding mode is in the bits 62:63 of FPCSR, and has 12760 // the following settings: 12761 // 00 Round to nearest 12762 // 01 Round to 0 12763 // 10 Round to +inf 12764 // 11 Round to -inf 12765 12766 // When the operand is immediate, using the two least significant bits of 12767 // the immediate to set the bits 62:63 of FPSCR. 12768 unsigned Mode = MI.getOperand(1).getImm(); 12769 BuildMI(*BB, MI, dl, TII->get((Mode & 1) ? PPC::MTFSB1 : PPC::MTFSB0)) 12770 .addImm(31) 12771 .addReg(PPC::RM, RegState::ImplicitDefine); 12772 12773 BuildMI(*BB, MI, dl, TII->get((Mode & 2) ? PPC::MTFSB1 : PPC::MTFSB0)) 12774 .addImm(30) 12775 .addReg(PPC::RM, RegState::ImplicitDefine); 12776 } else if (MI.getOpcode() == PPC::SETRND) { 12777 DebugLoc dl = MI.getDebugLoc(); 12778 12779 // Copy register from F8RCRegClass::SrcReg to G8RCRegClass::DestReg 12780 // or copy register from G8RCRegClass::SrcReg to F8RCRegClass::DestReg. 12781 // If the target doesn't have DirectMove, we should use stack to do the 12782 // conversion, because the target doesn't have the instructions like mtvsrd 12783 // or mfvsrd to do this conversion directly. 12784 auto copyRegFromG8RCOrF8RC = [&] (unsigned DestReg, unsigned SrcReg) { 12785 if (Subtarget.hasDirectMove()) { 12786 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), DestReg) 12787 .addReg(SrcReg); 12788 } else { 12789 // Use stack to do the register copy. 12790 unsigned StoreOp = PPC::STD, LoadOp = PPC::LFD; 12791 MachineRegisterInfo &RegInfo = F->getRegInfo(); 12792 const TargetRegisterClass *RC = RegInfo.getRegClass(SrcReg); 12793 if (RC == &PPC::F8RCRegClass) { 12794 // Copy register from F8RCRegClass to G8RCRegclass. 12795 assert((RegInfo.getRegClass(DestReg) == &PPC::G8RCRegClass) && 12796 "Unsupported RegClass."); 12797 12798 StoreOp = PPC::STFD; 12799 LoadOp = PPC::LD; 12800 } else { 12801 // Copy register from G8RCRegClass to F8RCRegclass. 12802 assert((RegInfo.getRegClass(SrcReg) == &PPC::G8RCRegClass) && 12803 (RegInfo.getRegClass(DestReg) == &PPC::F8RCRegClass) && 12804 "Unsupported RegClass."); 12805 } 12806 12807 MachineFrameInfo &MFI = F->getFrameInfo(); 12808 int FrameIdx = MFI.CreateStackObject(8, Align(8), false); 12809 12810 MachineMemOperand *MMOStore = F->getMachineMemOperand( 12811 MachinePointerInfo::getFixedStack(*F, FrameIdx, 0), 12812 MachineMemOperand::MOStore, MFI.getObjectSize(FrameIdx), 12813 MFI.getObjectAlign(FrameIdx)); 12814 12815 // Store the SrcReg into the stack. 12816 BuildMI(*BB, MI, dl, TII->get(StoreOp)) 12817 .addReg(SrcReg) 12818 .addImm(0) 12819 .addFrameIndex(FrameIdx) 12820 .addMemOperand(MMOStore); 12821 12822 MachineMemOperand *MMOLoad = F->getMachineMemOperand( 12823 MachinePointerInfo::getFixedStack(*F, FrameIdx, 0), 12824 MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIdx), 12825 MFI.getObjectAlign(FrameIdx)); 12826 12827 // Load from the stack where SrcReg is stored, and save to DestReg, 12828 // so we have done the RegClass conversion from RegClass::SrcReg to 12829 // RegClass::DestReg. 12830 BuildMI(*BB, MI, dl, TII->get(LoadOp), DestReg) 12831 .addImm(0) 12832 .addFrameIndex(FrameIdx) 12833 .addMemOperand(MMOLoad); 12834 } 12835 }; 12836 12837 Register OldFPSCRReg = MI.getOperand(0).getReg(); 12838 12839 // Save FPSCR value. 12840 BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), OldFPSCRReg); 12841 12842 // When the operand is gprc register, use two least significant bits of the 12843 // register and mtfsf instruction to set the bits 62:63 of FPSCR. 12844 // 12845 // copy OldFPSCRTmpReg, OldFPSCRReg 12846 // (INSERT_SUBREG ExtSrcReg, (IMPLICIT_DEF ImDefReg), SrcOp, 1) 12847 // rldimi NewFPSCRTmpReg, ExtSrcReg, OldFPSCRReg, 0, 62 12848 // copy NewFPSCRReg, NewFPSCRTmpReg 12849 // mtfsf 255, NewFPSCRReg 12850 MachineOperand SrcOp = MI.getOperand(1); 12851 MachineRegisterInfo &RegInfo = F->getRegInfo(); 12852 Register OldFPSCRTmpReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass); 12853 12854 copyRegFromG8RCOrF8RC(OldFPSCRTmpReg, OldFPSCRReg); 12855 12856 Register ImDefReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass); 12857 Register ExtSrcReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass); 12858 12859 // The first operand of INSERT_SUBREG should be a register which has 12860 // subregisters, we only care about its RegClass, so we should use an 12861 // IMPLICIT_DEF register. 12862 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::IMPLICIT_DEF), ImDefReg); 12863 BuildMI(*BB, MI, dl, TII->get(PPC::INSERT_SUBREG), ExtSrcReg) 12864 .addReg(ImDefReg) 12865 .add(SrcOp) 12866 .addImm(1); 12867 12868 Register NewFPSCRTmpReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass); 12869 BuildMI(*BB, MI, dl, TII->get(PPC::RLDIMI), NewFPSCRTmpReg) 12870 .addReg(OldFPSCRTmpReg) 12871 .addReg(ExtSrcReg) 12872 .addImm(0) 12873 .addImm(62); 12874 12875 Register NewFPSCRReg = RegInfo.createVirtualRegister(&PPC::F8RCRegClass); 12876 copyRegFromG8RCOrF8RC(NewFPSCRReg, NewFPSCRTmpReg); 12877 12878 // The mask 255 means that put the 32:63 bits of NewFPSCRReg to the 32:63 12879 // bits of FPSCR. 12880 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSF)) 12881 .addImm(255) 12882 .addReg(NewFPSCRReg) 12883 .addImm(0) 12884 .addImm(0); 12885 } else if (MI.getOpcode() == PPC::SETFLM) { 12886 DebugLoc Dl = MI.getDebugLoc(); 12887 12888 // Result of setflm is previous FPSCR content, so we need to save it first. 12889 Register OldFPSCRReg = MI.getOperand(0).getReg(); 12890 if (MRI.use_empty(OldFPSCRReg)) 12891 BuildMI(*BB, MI, Dl, TII->get(TargetOpcode::IMPLICIT_DEF), OldFPSCRReg); 12892 else 12893 BuildMI(*BB, MI, Dl, TII->get(PPC::MFFS), OldFPSCRReg); 12894 12895 // Put bits in 32:63 to FPSCR. 12896 Register NewFPSCRReg = MI.getOperand(1).getReg(); 12897 BuildMI(*BB, MI, Dl, TII->get(PPC::MTFSF)) 12898 .addImm(255) 12899 .addReg(NewFPSCRReg) 12900 .addImm(0) 12901 .addImm(0); 12902 } else if (MI.getOpcode() == PPC::PROBED_ALLOCA_32 || 12903 MI.getOpcode() == PPC::PROBED_ALLOCA_64) { 12904 return emitProbedAlloca(MI, BB); 12905 } else if (MI.getOpcode() == PPC::SPLIT_QUADWORD) { 12906 DebugLoc DL = MI.getDebugLoc(); 12907 Register Src = MI.getOperand(2).getReg(); 12908 Register Lo = MI.getOperand(0).getReg(); 12909 Register Hi = MI.getOperand(1).getReg(); 12910 BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY)) 12911 .addDef(Lo) 12912 .addUse(Src, 0, PPC::sub_gp8_x1); 12913 BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY)) 12914 .addDef(Hi) 12915 .addUse(Src, 0, PPC::sub_gp8_x0); 12916 } else if (MI.getOpcode() == PPC::LQX_PSEUDO || 12917 MI.getOpcode() == PPC::STQX_PSEUDO) { 12918 DebugLoc DL = MI.getDebugLoc(); 12919 // Ptr is used as the ptr_rc_no_r0 part 12920 // of LQ/STQ's memory operand and adding result of RA and RB, 12921 // so it has to be g8rc_and_g8rc_nox0. 12922 Register Ptr = 12923 F->getRegInfo().createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass); 12924 Register Val = MI.getOperand(0).getReg(); 12925 Register RA = MI.getOperand(1).getReg(); 12926 Register RB = MI.getOperand(2).getReg(); 12927 BuildMI(*BB, MI, DL, TII->get(PPC::ADD8), Ptr).addReg(RA).addReg(RB); 12928 BuildMI(*BB, MI, DL, 12929 MI.getOpcode() == PPC::LQX_PSEUDO ? TII->get(PPC::LQ) 12930 : TII->get(PPC::STQ)) 12931 .addReg(Val, MI.getOpcode() == PPC::LQX_PSEUDO ? RegState::Define : 0) 12932 .addImm(0) 12933 .addReg(Ptr); 12934 } else { 12935 llvm_unreachable("Unexpected instr type to insert"); 12936 } 12937 12938 MI.eraseFromParent(); // The pseudo instruction is gone now. 12939 return BB; 12940 } 12941 12942 //===----------------------------------------------------------------------===// 12943 // Target Optimization Hooks 12944 //===----------------------------------------------------------------------===// 12945 12946 static int getEstimateRefinementSteps(EVT VT, const PPCSubtarget &Subtarget) { 12947 // For the estimates, convergence is quadratic, so we essentially double the 12948 // number of digits correct after every iteration. For both FRE and FRSQRTE, 12949 // the minimum architected relative accuracy is 2^-5. When hasRecipPrec(), 12950 // this is 2^-14. IEEE float has 23 digits and double has 52 digits. 12951 int RefinementSteps = Subtarget.hasRecipPrec() ? 1 : 3; 12952 if (VT.getScalarType() == MVT::f64) 12953 RefinementSteps++; 12954 return RefinementSteps; 12955 } 12956 12957 SDValue PPCTargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG, 12958 const DenormalMode &Mode) const { 12959 // We only have VSX Vector Test for software Square Root. 12960 EVT VT = Op.getValueType(); 12961 if (!isTypeLegal(MVT::i1) || 12962 (VT != MVT::f64 && 12963 ((VT != MVT::v2f64 && VT != MVT::v4f32) || !Subtarget.hasVSX()))) 12964 return TargetLowering::getSqrtInputTest(Op, DAG, Mode); 12965 12966 SDLoc DL(Op); 12967 // The output register of FTSQRT is CR field. 12968 SDValue FTSQRT = DAG.getNode(PPCISD::FTSQRT, DL, MVT::i32, Op); 12969 // ftsqrt BF,FRB 12970 // Let e_b be the unbiased exponent of the double-precision 12971 // floating-point operand in register FRB. 12972 // fe_flag is set to 1 if either of the following conditions occurs. 12973 // - The double-precision floating-point operand in register FRB is a zero, 12974 // a NaN, or an infinity, or a negative value. 12975 // - e_b is less than or equal to -970. 12976 // Otherwise fe_flag is set to 0. 12977 // Both VSX and non-VSX versions would set EQ bit in the CR if the number is 12978 // not eligible for iteration. (zero/negative/infinity/nan or unbiased 12979 // exponent is less than -970) 12980 SDValue SRIdxVal = DAG.getTargetConstant(PPC::sub_eq, DL, MVT::i32); 12981 return SDValue(DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, MVT::i1, 12982 FTSQRT, SRIdxVal), 12983 0); 12984 } 12985 12986 SDValue 12987 PPCTargetLowering::getSqrtResultForDenormInput(SDValue Op, 12988 SelectionDAG &DAG) const { 12989 // We only have VSX Vector Square Root. 12990 EVT VT = Op.getValueType(); 12991 if (VT != MVT::f64 && 12992 ((VT != MVT::v2f64 && VT != MVT::v4f32) || !Subtarget.hasVSX())) 12993 return TargetLowering::getSqrtResultForDenormInput(Op, DAG); 12994 12995 return DAG.getNode(PPCISD::FSQRT, SDLoc(Op), VT, Op); 12996 } 12997 12998 SDValue PPCTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, 12999 int Enabled, int &RefinementSteps, 13000 bool &UseOneConstNR, 13001 bool Reciprocal) const { 13002 EVT VT = Operand.getValueType(); 13003 if ((VT == MVT::f32 && Subtarget.hasFRSQRTES()) || 13004 (VT == MVT::f64 && Subtarget.hasFRSQRTE()) || 13005 (VT == MVT::v4f32 && Subtarget.hasAltivec()) || 13006 (VT == MVT::v2f64 && Subtarget.hasVSX())) { 13007 if (RefinementSteps == ReciprocalEstimate::Unspecified) 13008 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget); 13009 13010 // The Newton-Raphson computation with a single constant does not provide 13011 // enough accuracy on some CPUs. 13012 UseOneConstNR = !Subtarget.needsTwoConstNR(); 13013 return DAG.getNode(PPCISD::FRSQRTE, SDLoc(Operand), VT, Operand); 13014 } 13015 return SDValue(); 13016 } 13017 13018 SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand, SelectionDAG &DAG, 13019 int Enabled, 13020 int &RefinementSteps) const { 13021 EVT VT = Operand.getValueType(); 13022 if ((VT == MVT::f32 && Subtarget.hasFRES()) || 13023 (VT == MVT::f64 && Subtarget.hasFRE()) || 13024 (VT == MVT::v4f32 && Subtarget.hasAltivec()) || 13025 (VT == MVT::v2f64 && Subtarget.hasVSX())) { 13026 if (RefinementSteps == ReciprocalEstimate::Unspecified) 13027 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget); 13028 return DAG.getNode(PPCISD::FRE, SDLoc(Operand), VT, Operand); 13029 } 13030 return SDValue(); 13031 } 13032 13033 unsigned PPCTargetLowering::combineRepeatedFPDivisors() const { 13034 // Note: This functionality is used only when unsafe-fp-math is enabled, and 13035 // on cores with reciprocal estimates (which are used when unsafe-fp-math is 13036 // enabled for division), this functionality is redundant with the default 13037 // combiner logic (once the division -> reciprocal/multiply transformation 13038 // has taken place). As a result, this matters more for older cores than for 13039 // newer ones. 13040 13041 // Combine multiple FDIVs with the same divisor into multiple FMULs by the 13042 // reciprocal if there are two or more FDIVs (for embedded cores with only 13043 // one FP pipeline) for three or more FDIVs (for generic OOO cores). 13044 switch (Subtarget.getCPUDirective()) { 13045 default: 13046 return 3; 13047 case PPC::DIR_440: 13048 case PPC::DIR_A2: 13049 case PPC::DIR_E500: 13050 case PPC::DIR_E500mc: 13051 case PPC::DIR_E5500: 13052 return 2; 13053 } 13054 } 13055 13056 // isConsecutiveLSLoc needs to work even if all adds have not yet been 13057 // collapsed, and so we need to look through chains of them. 13058 static void getBaseWithConstantOffset(SDValue Loc, SDValue &Base, 13059 int64_t& Offset, SelectionDAG &DAG) { 13060 if (DAG.isBaseWithConstantOffset(Loc)) { 13061 Base = Loc.getOperand(0); 13062 Offset += cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue(); 13063 13064 // The base might itself be a base plus an offset, and if so, accumulate 13065 // that as well. 13066 getBaseWithConstantOffset(Loc.getOperand(0), Base, Offset, DAG); 13067 } 13068 } 13069 13070 static bool isConsecutiveLSLoc(SDValue Loc, EVT VT, LSBaseSDNode *Base, 13071 unsigned Bytes, int Dist, 13072 SelectionDAG &DAG) { 13073 if (VT.getSizeInBits() / 8 != Bytes) 13074 return false; 13075 13076 SDValue BaseLoc = Base->getBasePtr(); 13077 if (Loc.getOpcode() == ISD::FrameIndex) { 13078 if (BaseLoc.getOpcode() != ISD::FrameIndex) 13079 return false; 13080 const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); 13081 int FI = cast<FrameIndexSDNode>(Loc)->getIndex(); 13082 int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex(); 13083 int FS = MFI.getObjectSize(FI); 13084 int BFS = MFI.getObjectSize(BFI); 13085 if (FS != BFS || FS != (int)Bytes) return false; 13086 return MFI.getObjectOffset(FI) == (MFI.getObjectOffset(BFI) + Dist*Bytes); 13087 } 13088 13089 SDValue Base1 = Loc, Base2 = BaseLoc; 13090 int64_t Offset1 = 0, Offset2 = 0; 13091 getBaseWithConstantOffset(Loc, Base1, Offset1, DAG); 13092 getBaseWithConstantOffset(BaseLoc, Base2, Offset2, DAG); 13093 if (Base1 == Base2 && Offset1 == (Offset2 + Dist * Bytes)) 13094 return true; 13095 13096 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 13097 const GlobalValue *GV1 = nullptr; 13098 const GlobalValue *GV2 = nullptr; 13099 Offset1 = 0; 13100 Offset2 = 0; 13101 bool isGA1 = TLI.isGAPlusOffset(Loc.getNode(), GV1, Offset1); 13102 bool isGA2 = TLI.isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2); 13103 if (isGA1 && isGA2 && GV1 == GV2) 13104 return Offset1 == (Offset2 + Dist*Bytes); 13105 return false; 13106 } 13107 13108 // Like SelectionDAG::isConsecutiveLoad, but also works for stores, and does 13109 // not enforce equality of the chain operands. 13110 static bool isConsecutiveLS(SDNode *N, LSBaseSDNode *Base, 13111 unsigned Bytes, int Dist, 13112 SelectionDAG &DAG) { 13113 if (LSBaseSDNode *LS = dyn_cast<LSBaseSDNode>(N)) { 13114 EVT VT = LS->getMemoryVT(); 13115 SDValue Loc = LS->getBasePtr(); 13116 return isConsecutiveLSLoc(Loc, VT, Base, Bytes, Dist, DAG); 13117 } 13118 13119 if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) { 13120 EVT VT; 13121 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) { 13122 default: return false; 13123 case Intrinsic::ppc_altivec_lvx: 13124 case Intrinsic::ppc_altivec_lvxl: 13125 case Intrinsic::ppc_vsx_lxvw4x: 13126 case Intrinsic::ppc_vsx_lxvw4x_be: 13127 VT = MVT::v4i32; 13128 break; 13129 case Intrinsic::ppc_vsx_lxvd2x: 13130 case Intrinsic::ppc_vsx_lxvd2x_be: 13131 VT = MVT::v2f64; 13132 break; 13133 case Intrinsic::ppc_altivec_lvebx: 13134 VT = MVT::i8; 13135 break; 13136 case Intrinsic::ppc_altivec_lvehx: 13137 VT = MVT::i16; 13138 break; 13139 case Intrinsic::ppc_altivec_lvewx: 13140 VT = MVT::i32; 13141 break; 13142 } 13143 13144 return isConsecutiveLSLoc(N->getOperand(2), VT, Base, Bytes, Dist, DAG); 13145 } 13146 13147 if (N->getOpcode() == ISD::INTRINSIC_VOID) { 13148 EVT VT; 13149 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) { 13150 default: return false; 13151 case Intrinsic::ppc_altivec_stvx: 13152 case Intrinsic::ppc_altivec_stvxl: 13153 case Intrinsic::ppc_vsx_stxvw4x: 13154 VT = MVT::v4i32; 13155 break; 13156 case Intrinsic::ppc_vsx_stxvd2x: 13157 VT = MVT::v2f64; 13158 break; 13159 case Intrinsic::ppc_vsx_stxvw4x_be: 13160 VT = MVT::v4i32; 13161 break; 13162 case Intrinsic::ppc_vsx_stxvd2x_be: 13163 VT = MVT::v2f64; 13164 break; 13165 case Intrinsic::ppc_altivec_stvebx: 13166 VT = MVT::i8; 13167 break; 13168 case Intrinsic::ppc_altivec_stvehx: 13169 VT = MVT::i16; 13170 break; 13171 case Intrinsic::ppc_altivec_stvewx: 13172 VT = MVT::i32; 13173 break; 13174 } 13175 13176 return isConsecutiveLSLoc(N->getOperand(3), VT, Base, Bytes, Dist, DAG); 13177 } 13178 13179 return false; 13180 } 13181 13182 // Return true is there is a nearyby consecutive load to the one provided 13183 // (regardless of alignment). We search up and down the chain, looking though 13184 // token factors and other loads (but nothing else). As a result, a true result 13185 // indicates that it is safe to create a new consecutive load adjacent to the 13186 // load provided. 13187 static bool findConsecutiveLoad(LoadSDNode *LD, SelectionDAG &DAG) { 13188 SDValue Chain = LD->getChain(); 13189 EVT VT = LD->getMemoryVT(); 13190 13191 SmallSet<SDNode *, 16> LoadRoots; 13192 SmallVector<SDNode *, 8> Queue(1, Chain.getNode()); 13193 SmallSet<SDNode *, 16> Visited; 13194 13195 // First, search up the chain, branching to follow all token-factor operands. 13196 // If we find a consecutive load, then we're done, otherwise, record all 13197 // nodes just above the top-level loads and token factors. 13198 while (!Queue.empty()) { 13199 SDNode *ChainNext = Queue.pop_back_val(); 13200 if (!Visited.insert(ChainNext).second) 13201 continue; 13202 13203 if (MemSDNode *ChainLD = dyn_cast<MemSDNode>(ChainNext)) { 13204 if (isConsecutiveLS(ChainLD, LD, VT.getStoreSize(), 1, DAG)) 13205 return true; 13206 13207 if (!Visited.count(ChainLD->getChain().getNode())) 13208 Queue.push_back(ChainLD->getChain().getNode()); 13209 } else if (ChainNext->getOpcode() == ISD::TokenFactor) { 13210 for (const SDUse &O : ChainNext->ops()) 13211 if (!Visited.count(O.getNode())) 13212 Queue.push_back(O.getNode()); 13213 } else 13214 LoadRoots.insert(ChainNext); 13215 } 13216 13217 // Second, search down the chain, starting from the top-level nodes recorded 13218 // in the first phase. These top-level nodes are the nodes just above all 13219 // loads and token factors. Starting with their uses, recursively look though 13220 // all loads (just the chain uses) and token factors to find a consecutive 13221 // load. 13222 Visited.clear(); 13223 Queue.clear(); 13224 13225 for (SmallSet<SDNode *, 16>::iterator I = LoadRoots.begin(), 13226 IE = LoadRoots.end(); I != IE; ++I) { 13227 Queue.push_back(*I); 13228 13229 while (!Queue.empty()) { 13230 SDNode *LoadRoot = Queue.pop_back_val(); 13231 if (!Visited.insert(LoadRoot).second) 13232 continue; 13233 13234 if (MemSDNode *ChainLD = dyn_cast<MemSDNode>(LoadRoot)) 13235 if (isConsecutiveLS(ChainLD, LD, VT.getStoreSize(), 1, DAG)) 13236 return true; 13237 13238 for (SDNode *U : LoadRoot->uses()) 13239 if (((isa<MemSDNode>(U) && 13240 cast<MemSDNode>(U)->getChain().getNode() == LoadRoot) || 13241 U->getOpcode() == ISD::TokenFactor) && 13242 !Visited.count(U)) 13243 Queue.push_back(U); 13244 } 13245 } 13246 13247 return false; 13248 } 13249 13250 /// This function is called when we have proved that a SETCC node can be replaced 13251 /// by subtraction (and other supporting instructions) so that the result of 13252 /// comparison is kept in a GPR instead of CR. This function is purely for 13253 /// codegen purposes and has some flags to guide the codegen process. 13254 static SDValue generateEquivalentSub(SDNode *N, int Size, bool Complement, 13255 bool Swap, SDLoc &DL, SelectionDAG &DAG) { 13256 assert(N->getOpcode() == ISD::SETCC && "ISD::SETCC Expected."); 13257 13258 // Zero extend the operands to the largest legal integer. Originally, they 13259 // must be of a strictly smaller size. 13260 auto Op0 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(0), 13261 DAG.getConstant(Size, DL, MVT::i32)); 13262 auto Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1), 13263 DAG.getConstant(Size, DL, MVT::i32)); 13264 13265 // Swap if needed. Depends on the condition code. 13266 if (Swap) 13267 std::swap(Op0, Op1); 13268 13269 // Subtract extended integers. 13270 auto SubNode = DAG.getNode(ISD::SUB, DL, MVT::i64, Op0, Op1); 13271 13272 // Move the sign bit to the least significant position and zero out the rest. 13273 // Now the least significant bit carries the result of original comparison. 13274 auto Shifted = DAG.getNode(ISD::SRL, DL, MVT::i64, SubNode, 13275 DAG.getConstant(Size - 1, DL, MVT::i32)); 13276 auto Final = Shifted; 13277 13278 // Complement the result if needed. Based on the condition code. 13279 if (Complement) 13280 Final = DAG.getNode(ISD::XOR, DL, MVT::i64, Shifted, 13281 DAG.getConstant(1, DL, MVT::i64)); 13282 13283 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Final); 13284 } 13285 13286 SDValue PPCTargetLowering::ConvertSETCCToSubtract(SDNode *N, 13287 DAGCombinerInfo &DCI) const { 13288 assert(N->getOpcode() == ISD::SETCC && "ISD::SETCC Expected."); 13289 13290 SelectionDAG &DAG = DCI.DAG; 13291 SDLoc DL(N); 13292 13293 // Size of integers being compared has a critical role in the following 13294 // analysis, so we prefer to do this when all types are legal. 13295 if (!DCI.isAfterLegalizeDAG()) 13296 return SDValue(); 13297 13298 // If all users of SETCC extend its value to a legal integer type 13299 // then we replace SETCC with a subtraction 13300 for (const SDNode *U : N->uses()) 13301 if (U->getOpcode() != ISD::ZERO_EXTEND) 13302 return SDValue(); 13303 13304 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get(); 13305 auto OpSize = N->getOperand(0).getValueSizeInBits(); 13306 13307 unsigned Size = DAG.getDataLayout().getLargestLegalIntTypeSizeInBits(); 13308 13309 if (OpSize < Size) { 13310 switch (CC) { 13311 default: break; 13312 case ISD::SETULT: 13313 return generateEquivalentSub(N, Size, false, false, DL, DAG); 13314 case ISD::SETULE: 13315 return generateEquivalentSub(N, Size, true, true, DL, DAG); 13316 case ISD::SETUGT: 13317 return generateEquivalentSub(N, Size, false, true, DL, DAG); 13318 case ISD::SETUGE: 13319 return generateEquivalentSub(N, Size, true, false, DL, DAG); 13320 } 13321 } 13322 13323 return SDValue(); 13324 } 13325 13326 SDValue PPCTargetLowering::DAGCombineTruncBoolExt(SDNode *N, 13327 DAGCombinerInfo &DCI) const { 13328 SelectionDAG &DAG = DCI.DAG; 13329 SDLoc dl(N); 13330 13331 assert(Subtarget.useCRBits() && "Expecting to be tracking CR bits"); 13332 // If we're tracking CR bits, we need to be careful that we don't have: 13333 // trunc(binary-ops(zext(x), zext(y))) 13334 // or 13335 // trunc(binary-ops(binary-ops(zext(x), zext(y)), ...) 13336 // such that we're unnecessarily moving things into GPRs when it would be 13337 // better to keep them in CR bits. 13338 13339 // Note that trunc here can be an actual i1 trunc, or can be the effective 13340 // truncation that comes from a setcc or select_cc. 13341 if (N->getOpcode() == ISD::TRUNCATE && 13342 N->getValueType(0) != MVT::i1) 13343 return SDValue(); 13344 13345 if (N->getOperand(0).getValueType() != MVT::i32 && 13346 N->getOperand(0).getValueType() != MVT::i64) 13347 return SDValue(); 13348 13349 if (N->getOpcode() == ISD::SETCC || 13350 N->getOpcode() == ISD::SELECT_CC) { 13351 // If we're looking at a comparison, then we need to make sure that the 13352 // high bits (all except for the first) don't matter the result. 13353 ISD::CondCode CC = 13354 cast<CondCodeSDNode>(N->getOperand( 13355 N->getOpcode() == ISD::SETCC ? 2 : 4))->get(); 13356 unsigned OpBits = N->getOperand(0).getValueSizeInBits(); 13357 13358 if (ISD::isSignedIntSetCC(CC)) { 13359 if (DAG.ComputeNumSignBits(N->getOperand(0)) != OpBits || 13360 DAG.ComputeNumSignBits(N->getOperand(1)) != OpBits) 13361 return SDValue(); 13362 } else if (ISD::isUnsignedIntSetCC(CC)) { 13363 if (!DAG.MaskedValueIsZero(N->getOperand(0), 13364 APInt::getHighBitsSet(OpBits, OpBits-1)) || 13365 !DAG.MaskedValueIsZero(N->getOperand(1), 13366 APInt::getHighBitsSet(OpBits, OpBits-1))) 13367 return (N->getOpcode() == ISD::SETCC ? ConvertSETCCToSubtract(N, DCI) 13368 : SDValue()); 13369 } else { 13370 // This is neither a signed nor an unsigned comparison, just make sure 13371 // that the high bits are equal. 13372 KnownBits Op1Known = DAG.computeKnownBits(N->getOperand(0)); 13373 KnownBits Op2Known = DAG.computeKnownBits(N->getOperand(1)); 13374 13375 // We don't really care about what is known about the first bit (if 13376 // anything), so pretend that it is known zero for both to ensure they can 13377 // be compared as constants. 13378 Op1Known.Zero.setBit(0); Op1Known.One.clearBit(0); 13379 Op2Known.Zero.setBit(0); Op2Known.One.clearBit(0); 13380 13381 if (!Op1Known.isConstant() || !Op2Known.isConstant() || 13382 Op1Known.getConstant() != Op2Known.getConstant()) 13383 return SDValue(); 13384 } 13385 } 13386 13387 // We now know that the higher-order bits are irrelevant, we just need to 13388 // make sure that all of the intermediate operations are bit operations, and 13389 // all inputs are extensions. 13390 if (N->getOperand(0).getOpcode() != ISD::AND && 13391 N->getOperand(0).getOpcode() != ISD::OR && 13392 N->getOperand(0).getOpcode() != ISD::XOR && 13393 N->getOperand(0).getOpcode() != ISD::SELECT && 13394 N->getOperand(0).getOpcode() != ISD::SELECT_CC && 13395 N->getOperand(0).getOpcode() != ISD::TRUNCATE && 13396 N->getOperand(0).getOpcode() != ISD::SIGN_EXTEND && 13397 N->getOperand(0).getOpcode() != ISD::ZERO_EXTEND && 13398 N->getOperand(0).getOpcode() != ISD::ANY_EXTEND) 13399 return SDValue(); 13400 13401 if ((N->getOpcode() == ISD::SETCC || N->getOpcode() == ISD::SELECT_CC) && 13402 N->getOperand(1).getOpcode() != ISD::AND && 13403 N->getOperand(1).getOpcode() != ISD::OR && 13404 N->getOperand(1).getOpcode() != ISD::XOR && 13405 N->getOperand(1).getOpcode() != ISD::SELECT && 13406 N->getOperand(1).getOpcode() != ISD::SELECT_CC && 13407 N->getOperand(1).getOpcode() != ISD::TRUNCATE && 13408 N->getOperand(1).getOpcode() != ISD::SIGN_EXTEND && 13409 N->getOperand(1).getOpcode() != ISD::ZERO_EXTEND && 13410 N->getOperand(1).getOpcode() != ISD::ANY_EXTEND) 13411 return SDValue(); 13412 13413 SmallVector<SDValue, 4> Inputs; 13414 SmallVector<SDValue, 8> BinOps, PromOps; 13415 SmallPtrSet<SDNode *, 16> Visited; 13416 13417 for (unsigned i = 0; i < 2; ++i) { 13418 if (((N->getOperand(i).getOpcode() == ISD::SIGN_EXTEND || 13419 N->getOperand(i).getOpcode() == ISD::ZERO_EXTEND || 13420 N->getOperand(i).getOpcode() == ISD::ANY_EXTEND) && 13421 N->getOperand(i).getOperand(0).getValueType() == MVT::i1) || 13422 isa<ConstantSDNode>(N->getOperand(i))) 13423 Inputs.push_back(N->getOperand(i)); 13424 else 13425 BinOps.push_back(N->getOperand(i)); 13426 13427 if (N->getOpcode() == ISD::TRUNCATE) 13428 break; 13429 } 13430 13431 // Visit all inputs, collect all binary operations (and, or, xor and 13432 // select) that are all fed by extensions. 13433 while (!BinOps.empty()) { 13434 SDValue BinOp = BinOps.pop_back_val(); 13435 13436 if (!Visited.insert(BinOp.getNode()).second) 13437 continue; 13438 13439 PromOps.push_back(BinOp); 13440 13441 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) { 13442 // The condition of the select is not promoted. 13443 if (BinOp.getOpcode() == ISD::SELECT && i == 0) 13444 continue; 13445 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3) 13446 continue; 13447 13448 if (((BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND || 13449 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND || 13450 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) && 13451 BinOp.getOperand(i).getOperand(0).getValueType() == MVT::i1) || 13452 isa<ConstantSDNode>(BinOp.getOperand(i))) { 13453 Inputs.push_back(BinOp.getOperand(i)); 13454 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND || 13455 BinOp.getOperand(i).getOpcode() == ISD::OR || 13456 BinOp.getOperand(i).getOpcode() == ISD::XOR || 13457 BinOp.getOperand(i).getOpcode() == ISD::SELECT || 13458 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC || 13459 BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE || 13460 BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND || 13461 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND || 13462 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) { 13463 BinOps.push_back(BinOp.getOperand(i)); 13464 } else { 13465 // We have an input that is not an extension or another binary 13466 // operation; we'll abort this transformation. 13467 return SDValue(); 13468 } 13469 } 13470 } 13471 13472 // Make sure that this is a self-contained cluster of operations (which 13473 // is not quite the same thing as saying that everything has only one 13474 // use). 13475 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 13476 if (isa<ConstantSDNode>(Inputs[i])) 13477 continue; 13478 13479 for (const SDNode *User : Inputs[i].getNode()->uses()) { 13480 if (User != N && !Visited.count(User)) 13481 return SDValue(); 13482 13483 // Make sure that we're not going to promote the non-output-value 13484 // operand(s) or SELECT or SELECT_CC. 13485 // FIXME: Although we could sometimes handle this, and it does occur in 13486 // practice that one of the condition inputs to the select is also one of 13487 // the outputs, we currently can't deal with this. 13488 if (User->getOpcode() == ISD::SELECT) { 13489 if (User->getOperand(0) == Inputs[i]) 13490 return SDValue(); 13491 } else if (User->getOpcode() == ISD::SELECT_CC) { 13492 if (User->getOperand(0) == Inputs[i] || 13493 User->getOperand(1) == Inputs[i]) 13494 return SDValue(); 13495 } 13496 } 13497 } 13498 13499 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) { 13500 for (const SDNode *User : PromOps[i].getNode()->uses()) { 13501 if (User != N && !Visited.count(User)) 13502 return SDValue(); 13503 13504 // Make sure that we're not going to promote the non-output-value 13505 // operand(s) or SELECT or SELECT_CC. 13506 // FIXME: Although we could sometimes handle this, and it does occur in 13507 // practice that one of the condition inputs to the select is also one of 13508 // the outputs, we currently can't deal with this. 13509 if (User->getOpcode() == ISD::SELECT) { 13510 if (User->getOperand(0) == PromOps[i]) 13511 return SDValue(); 13512 } else if (User->getOpcode() == ISD::SELECT_CC) { 13513 if (User->getOperand(0) == PromOps[i] || 13514 User->getOperand(1) == PromOps[i]) 13515 return SDValue(); 13516 } 13517 } 13518 } 13519 13520 // Replace all inputs with the extension operand. 13521 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 13522 // Constants may have users outside the cluster of to-be-promoted nodes, 13523 // and so we need to replace those as we do the promotions. 13524 if (isa<ConstantSDNode>(Inputs[i])) 13525 continue; 13526 else 13527 DAG.ReplaceAllUsesOfValueWith(Inputs[i], Inputs[i].getOperand(0)); 13528 } 13529 13530 std::list<HandleSDNode> PromOpHandles; 13531 for (auto &PromOp : PromOps) 13532 PromOpHandles.emplace_back(PromOp); 13533 13534 // Replace all operations (these are all the same, but have a different 13535 // (i1) return type). DAG.getNode will validate that the types of 13536 // a binary operator match, so go through the list in reverse so that 13537 // we've likely promoted both operands first. Any intermediate truncations or 13538 // extensions disappear. 13539 while (!PromOpHandles.empty()) { 13540 SDValue PromOp = PromOpHandles.back().getValue(); 13541 PromOpHandles.pop_back(); 13542 13543 if (PromOp.getOpcode() == ISD::TRUNCATE || 13544 PromOp.getOpcode() == ISD::SIGN_EXTEND || 13545 PromOp.getOpcode() == ISD::ZERO_EXTEND || 13546 PromOp.getOpcode() == ISD::ANY_EXTEND) { 13547 if (!isa<ConstantSDNode>(PromOp.getOperand(0)) && 13548 PromOp.getOperand(0).getValueType() != MVT::i1) { 13549 // The operand is not yet ready (see comment below). 13550 PromOpHandles.emplace_front(PromOp); 13551 continue; 13552 } 13553 13554 SDValue RepValue = PromOp.getOperand(0); 13555 if (isa<ConstantSDNode>(RepValue)) 13556 RepValue = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, RepValue); 13557 13558 DAG.ReplaceAllUsesOfValueWith(PromOp, RepValue); 13559 continue; 13560 } 13561 13562 unsigned C; 13563 switch (PromOp.getOpcode()) { 13564 default: C = 0; break; 13565 case ISD::SELECT: C = 1; break; 13566 case ISD::SELECT_CC: C = 2; break; 13567 } 13568 13569 if ((!isa<ConstantSDNode>(PromOp.getOperand(C)) && 13570 PromOp.getOperand(C).getValueType() != MVT::i1) || 13571 (!isa<ConstantSDNode>(PromOp.getOperand(C+1)) && 13572 PromOp.getOperand(C+1).getValueType() != MVT::i1)) { 13573 // The to-be-promoted operands of this node have not yet been 13574 // promoted (this should be rare because we're going through the 13575 // list backward, but if one of the operands has several users in 13576 // this cluster of to-be-promoted nodes, it is possible). 13577 PromOpHandles.emplace_front(PromOp); 13578 continue; 13579 } 13580 13581 SmallVector<SDValue, 3> Ops(PromOp.getNode()->op_begin(), 13582 PromOp.getNode()->op_end()); 13583 13584 // If there are any constant inputs, make sure they're replaced now. 13585 for (unsigned i = 0; i < 2; ++i) 13586 if (isa<ConstantSDNode>(Ops[C+i])) 13587 Ops[C+i] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Ops[C+i]); 13588 13589 DAG.ReplaceAllUsesOfValueWith(PromOp, 13590 DAG.getNode(PromOp.getOpcode(), dl, MVT::i1, Ops)); 13591 } 13592 13593 // Now we're left with the initial truncation itself. 13594 if (N->getOpcode() == ISD::TRUNCATE) 13595 return N->getOperand(0); 13596 13597 // Otherwise, this is a comparison. The operands to be compared have just 13598 // changed type (to i1), but everything else is the same. 13599 return SDValue(N, 0); 13600 } 13601 13602 SDValue PPCTargetLowering::DAGCombineExtBoolTrunc(SDNode *N, 13603 DAGCombinerInfo &DCI) const { 13604 SelectionDAG &DAG = DCI.DAG; 13605 SDLoc dl(N); 13606 13607 // If we're tracking CR bits, we need to be careful that we don't have: 13608 // zext(binary-ops(trunc(x), trunc(y))) 13609 // or 13610 // zext(binary-ops(binary-ops(trunc(x), trunc(y)), ...) 13611 // such that we're unnecessarily moving things into CR bits that can more 13612 // efficiently stay in GPRs. Note that if we're not certain that the high 13613 // bits are set as required by the final extension, we still may need to do 13614 // some masking to get the proper behavior. 13615 13616 // This same functionality is important on PPC64 when dealing with 13617 // 32-to-64-bit extensions; these occur often when 32-bit values are used as 13618 // the return values of functions. Because it is so similar, it is handled 13619 // here as well. 13620 13621 if (N->getValueType(0) != MVT::i32 && 13622 N->getValueType(0) != MVT::i64) 13623 return SDValue(); 13624 13625 if (!((N->getOperand(0).getValueType() == MVT::i1 && Subtarget.useCRBits()) || 13626 (N->getOperand(0).getValueType() == MVT::i32 && Subtarget.isPPC64()))) 13627 return SDValue(); 13628 13629 if (N->getOperand(0).getOpcode() != ISD::AND && 13630 N->getOperand(0).getOpcode() != ISD::OR && 13631 N->getOperand(0).getOpcode() != ISD::XOR && 13632 N->getOperand(0).getOpcode() != ISD::SELECT && 13633 N->getOperand(0).getOpcode() != ISD::SELECT_CC) 13634 return SDValue(); 13635 13636 SmallVector<SDValue, 4> Inputs; 13637 SmallVector<SDValue, 8> BinOps(1, N->getOperand(0)), PromOps; 13638 SmallPtrSet<SDNode *, 16> Visited; 13639 13640 // Visit all inputs, collect all binary operations (and, or, xor and 13641 // select) that are all fed by truncations. 13642 while (!BinOps.empty()) { 13643 SDValue BinOp = BinOps.pop_back_val(); 13644 13645 if (!Visited.insert(BinOp.getNode()).second) 13646 continue; 13647 13648 PromOps.push_back(BinOp); 13649 13650 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) { 13651 // The condition of the select is not promoted. 13652 if (BinOp.getOpcode() == ISD::SELECT && i == 0) 13653 continue; 13654 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3) 13655 continue; 13656 13657 if (BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE || 13658 isa<ConstantSDNode>(BinOp.getOperand(i))) { 13659 Inputs.push_back(BinOp.getOperand(i)); 13660 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND || 13661 BinOp.getOperand(i).getOpcode() == ISD::OR || 13662 BinOp.getOperand(i).getOpcode() == ISD::XOR || 13663 BinOp.getOperand(i).getOpcode() == ISD::SELECT || 13664 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC) { 13665 BinOps.push_back(BinOp.getOperand(i)); 13666 } else { 13667 // We have an input that is not a truncation or another binary 13668 // operation; we'll abort this transformation. 13669 return SDValue(); 13670 } 13671 } 13672 } 13673 13674 // The operands of a select that must be truncated when the select is 13675 // promoted because the operand is actually part of the to-be-promoted set. 13676 DenseMap<SDNode *, EVT> SelectTruncOp[2]; 13677 13678 // Make sure that this is a self-contained cluster of operations (which 13679 // is not quite the same thing as saying that everything has only one 13680 // use). 13681 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 13682 if (isa<ConstantSDNode>(Inputs[i])) 13683 continue; 13684 13685 for (SDNode *User : Inputs[i].getNode()->uses()) { 13686 if (User != N && !Visited.count(User)) 13687 return SDValue(); 13688 13689 // If we're going to promote the non-output-value operand(s) or SELECT or 13690 // SELECT_CC, record them for truncation. 13691 if (User->getOpcode() == ISD::SELECT) { 13692 if (User->getOperand(0) == Inputs[i]) 13693 SelectTruncOp[0].insert(std::make_pair(User, 13694 User->getOperand(0).getValueType())); 13695 } else if (User->getOpcode() == ISD::SELECT_CC) { 13696 if (User->getOperand(0) == Inputs[i]) 13697 SelectTruncOp[0].insert(std::make_pair(User, 13698 User->getOperand(0).getValueType())); 13699 if (User->getOperand(1) == Inputs[i]) 13700 SelectTruncOp[1].insert(std::make_pair(User, 13701 User->getOperand(1).getValueType())); 13702 } 13703 } 13704 } 13705 13706 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) { 13707 for (SDNode *User : PromOps[i].getNode()->uses()) { 13708 if (User != N && !Visited.count(User)) 13709 return SDValue(); 13710 13711 // If we're going to promote the non-output-value operand(s) or SELECT or 13712 // SELECT_CC, record them for truncation. 13713 if (User->getOpcode() == ISD::SELECT) { 13714 if (User->getOperand(0) == PromOps[i]) 13715 SelectTruncOp[0].insert(std::make_pair(User, 13716 User->getOperand(0).getValueType())); 13717 } else if (User->getOpcode() == ISD::SELECT_CC) { 13718 if (User->getOperand(0) == PromOps[i]) 13719 SelectTruncOp[0].insert(std::make_pair(User, 13720 User->getOperand(0).getValueType())); 13721 if (User->getOperand(1) == PromOps[i]) 13722 SelectTruncOp[1].insert(std::make_pair(User, 13723 User->getOperand(1).getValueType())); 13724 } 13725 } 13726 } 13727 13728 unsigned PromBits = N->getOperand(0).getValueSizeInBits(); 13729 bool ReallyNeedsExt = false; 13730 if (N->getOpcode() != ISD::ANY_EXTEND) { 13731 // If all of the inputs are not already sign/zero extended, then 13732 // we'll still need to do that at the end. 13733 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 13734 if (isa<ConstantSDNode>(Inputs[i])) 13735 continue; 13736 13737 unsigned OpBits = 13738 Inputs[i].getOperand(0).getValueSizeInBits(); 13739 assert(PromBits < OpBits && "Truncation not to a smaller bit count?"); 13740 13741 if ((N->getOpcode() == ISD::ZERO_EXTEND && 13742 !DAG.MaskedValueIsZero(Inputs[i].getOperand(0), 13743 APInt::getHighBitsSet(OpBits, 13744 OpBits-PromBits))) || 13745 (N->getOpcode() == ISD::SIGN_EXTEND && 13746 DAG.ComputeNumSignBits(Inputs[i].getOperand(0)) < 13747 (OpBits-(PromBits-1)))) { 13748 ReallyNeedsExt = true; 13749 break; 13750 } 13751 } 13752 } 13753 13754 // Replace all inputs, either with the truncation operand, or a 13755 // truncation or extension to the final output type. 13756 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 13757 // Constant inputs need to be replaced with the to-be-promoted nodes that 13758 // use them because they might have users outside of the cluster of 13759 // promoted nodes. 13760 if (isa<ConstantSDNode>(Inputs[i])) 13761 continue; 13762 13763 SDValue InSrc = Inputs[i].getOperand(0); 13764 if (Inputs[i].getValueType() == N->getValueType(0)) 13765 DAG.ReplaceAllUsesOfValueWith(Inputs[i], InSrc); 13766 else if (N->getOpcode() == ISD::SIGN_EXTEND) 13767 DAG.ReplaceAllUsesOfValueWith(Inputs[i], 13768 DAG.getSExtOrTrunc(InSrc, dl, N->getValueType(0))); 13769 else if (N->getOpcode() == ISD::ZERO_EXTEND) 13770 DAG.ReplaceAllUsesOfValueWith(Inputs[i], 13771 DAG.getZExtOrTrunc(InSrc, dl, N->getValueType(0))); 13772 else 13773 DAG.ReplaceAllUsesOfValueWith(Inputs[i], 13774 DAG.getAnyExtOrTrunc(InSrc, dl, N->getValueType(0))); 13775 } 13776 13777 std::list<HandleSDNode> PromOpHandles; 13778 for (auto &PromOp : PromOps) 13779 PromOpHandles.emplace_back(PromOp); 13780 13781 // Replace all operations (these are all the same, but have a different 13782 // (promoted) return type). DAG.getNode will validate that the types of 13783 // a binary operator match, so go through the list in reverse so that 13784 // we've likely promoted both operands first. 13785 while (!PromOpHandles.empty()) { 13786 SDValue PromOp = PromOpHandles.back().getValue(); 13787 PromOpHandles.pop_back(); 13788 13789 unsigned C; 13790 switch (PromOp.getOpcode()) { 13791 default: C = 0; break; 13792 case ISD::SELECT: C = 1; break; 13793 case ISD::SELECT_CC: C = 2; break; 13794 } 13795 13796 if ((!isa<ConstantSDNode>(PromOp.getOperand(C)) && 13797 PromOp.getOperand(C).getValueType() != N->getValueType(0)) || 13798 (!isa<ConstantSDNode>(PromOp.getOperand(C+1)) && 13799 PromOp.getOperand(C+1).getValueType() != N->getValueType(0))) { 13800 // The to-be-promoted operands of this node have not yet been 13801 // promoted (this should be rare because we're going through the 13802 // list backward, but if one of the operands has several users in 13803 // this cluster of to-be-promoted nodes, it is possible). 13804 PromOpHandles.emplace_front(PromOp); 13805 continue; 13806 } 13807 13808 // For SELECT and SELECT_CC nodes, we do a similar check for any 13809 // to-be-promoted comparison inputs. 13810 if (PromOp.getOpcode() == ISD::SELECT || 13811 PromOp.getOpcode() == ISD::SELECT_CC) { 13812 if ((SelectTruncOp[0].count(PromOp.getNode()) && 13813 PromOp.getOperand(0).getValueType() != N->getValueType(0)) || 13814 (SelectTruncOp[1].count(PromOp.getNode()) && 13815 PromOp.getOperand(1).getValueType() != N->getValueType(0))) { 13816 PromOpHandles.emplace_front(PromOp); 13817 continue; 13818 } 13819 } 13820 13821 SmallVector<SDValue, 3> Ops(PromOp.getNode()->op_begin(), 13822 PromOp.getNode()->op_end()); 13823 13824 // If this node has constant inputs, then they'll need to be promoted here. 13825 for (unsigned i = 0; i < 2; ++i) { 13826 if (!isa<ConstantSDNode>(Ops[C+i])) 13827 continue; 13828 if (Ops[C+i].getValueType() == N->getValueType(0)) 13829 continue; 13830 13831 if (N->getOpcode() == ISD::SIGN_EXTEND) 13832 Ops[C+i] = DAG.getSExtOrTrunc(Ops[C+i], dl, N->getValueType(0)); 13833 else if (N->getOpcode() == ISD::ZERO_EXTEND) 13834 Ops[C+i] = DAG.getZExtOrTrunc(Ops[C+i], dl, N->getValueType(0)); 13835 else 13836 Ops[C+i] = DAG.getAnyExtOrTrunc(Ops[C+i], dl, N->getValueType(0)); 13837 } 13838 13839 // If we've promoted the comparison inputs of a SELECT or SELECT_CC, 13840 // truncate them again to the original value type. 13841 if (PromOp.getOpcode() == ISD::SELECT || 13842 PromOp.getOpcode() == ISD::SELECT_CC) { 13843 auto SI0 = SelectTruncOp[0].find(PromOp.getNode()); 13844 if (SI0 != SelectTruncOp[0].end()) 13845 Ops[0] = DAG.getNode(ISD::TRUNCATE, dl, SI0->second, Ops[0]); 13846 auto SI1 = SelectTruncOp[1].find(PromOp.getNode()); 13847 if (SI1 != SelectTruncOp[1].end()) 13848 Ops[1] = DAG.getNode(ISD::TRUNCATE, dl, SI1->second, Ops[1]); 13849 } 13850 13851 DAG.ReplaceAllUsesOfValueWith(PromOp, 13852 DAG.getNode(PromOp.getOpcode(), dl, N->getValueType(0), Ops)); 13853 } 13854 13855 // Now we're left with the initial extension itself. 13856 if (!ReallyNeedsExt) 13857 return N->getOperand(0); 13858 13859 // To zero extend, just mask off everything except for the first bit (in the 13860 // i1 case). 13861 if (N->getOpcode() == ISD::ZERO_EXTEND) 13862 return DAG.getNode(ISD::AND, dl, N->getValueType(0), N->getOperand(0), 13863 DAG.getConstant(APInt::getLowBitsSet( 13864 N->getValueSizeInBits(0), PromBits), 13865 dl, N->getValueType(0))); 13866 13867 assert(N->getOpcode() == ISD::SIGN_EXTEND && 13868 "Invalid extension type"); 13869 EVT ShiftAmountTy = getShiftAmountTy(N->getValueType(0), DAG.getDataLayout()); 13870 SDValue ShiftCst = 13871 DAG.getConstant(N->getValueSizeInBits(0) - PromBits, dl, ShiftAmountTy); 13872 return DAG.getNode( 13873 ISD::SRA, dl, N->getValueType(0), 13874 DAG.getNode(ISD::SHL, dl, N->getValueType(0), N->getOperand(0), ShiftCst), 13875 ShiftCst); 13876 } 13877 13878 SDValue PPCTargetLowering::combineSetCC(SDNode *N, 13879 DAGCombinerInfo &DCI) const { 13880 assert(N->getOpcode() == ISD::SETCC && 13881 "Should be called with a SETCC node"); 13882 13883 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get(); 13884 if (CC == ISD::SETNE || CC == ISD::SETEQ) { 13885 SDValue LHS = N->getOperand(0); 13886 SDValue RHS = N->getOperand(1); 13887 13888 // If there is a '0 - y' pattern, canonicalize the pattern to the RHS. 13889 if (LHS.getOpcode() == ISD::SUB && isNullConstant(LHS.getOperand(0)) && 13890 LHS.hasOneUse()) 13891 std::swap(LHS, RHS); 13892 13893 // x == 0-y --> x+y == 0 13894 // x != 0-y --> x+y != 0 13895 if (RHS.getOpcode() == ISD::SUB && isNullConstant(RHS.getOperand(0)) && 13896 RHS.hasOneUse()) { 13897 SDLoc DL(N); 13898 SelectionDAG &DAG = DCI.DAG; 13899 EVT VT = N->getValueType(0); 13900 EVT OpVT = LHS.getValueType(); 13901 SDValue Add = DAG.getNode(ISD::ADD, DL, OpVT, LHS, RHS.getOperand(1)); 13902 return DAG.getSetCC(DL, VT, Add, DAG.getConstant(0, DL, OpVT), CC); 13903 } 13904 } 13905 13906 return DAGCombineTruncBoolExt(N, DCI); 13907 } 13908 13909 // Is this an extending load from an f32 to an f64? 13910 static bool isFPExtLoad(SDValue Op) { 13911 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op.getNode())) 13912 return LD->getExtensionType() == ISD::EXTLOAD && 13913 Op.getValueType() == MVT::f64; 13914 return false; 13915 } 13916 13917 /// Reduces the number of fp-to-int conversion when building a vector. 13918 /// 13919 /// If this vector is built out of floating to integer conversions, 13920 /// transform it to a vector built out of floating point values followed by a 13921 /// single floating to integer conversion of the vector. 13922 /// Namely (build_vector (fptosi $A), (fptosi $B), ...) 13923 /// becomes (fptosi (build_vector ($A, $B, ...))) 13924 SDValue PPCTargetLowering:: 13925 combineElementTruncationToVectorTruncation(SDNode *N, 13926 DAGCombinerInfo &DCI) const { 13927 assert(N->getOpcode() == ISD::BUILD_VECTOR && 13928 "Should be called with a BUILD_VECTOR node"); 13929 13930 SelectionDAG &DAG = DCI.DAG; 13931 SDLoc dl(N); 13932 13933 SDValue FirstInput = N->getOperand(0); 13934 assert(FirstInput.getOpcode() == PPCISD::MFVSR && 13935 "The input operand must be an fp-to-int conversion."); 13936 13937 // This combine happens after legalization so the fp_to_[su]i nodes are 13938 // already converted to PPCSISD nodes. 13939 unsigned FirstConversion = FirstInput.getOperand(0).getOpcode(); 13940 if (FirstConversion == PPCISD::FCTIDZ || 13941 FirstConversion == PPCISD::FCTIDUZ || 13942 FirstConversion == PPCISD::FCTIWZ || 13943 FirstConversion == PPCISD::FCTIWUZ) { 13944 bool IsSplat = true; 13945 bool Is32Bit = FirstConversion == PPCISD::FCTIWZ || 13946 FirstConversion == PPCISD::FCTIWUZ; 13947 EVT SrcVT = FirstInput.getOperand(0).getValueType(); 13948 SmallVector<SDValue, 4> Ops; 13949 EVT TargetVT = N->getValueType(0); 13950 for (int i = 0, e = N->getNumOperands(); i < e; ++i) { 13951 SDValue NextOp = N->getOperand(i); 13952 if (NextOp.getOpcode() != PPCISD::MFVSR) 13953 return SDValue(); 13954 unsigned NextConversion = NextOp.getOperand(0).getOpcode(); 13955 if (NextConversion != FirstConversion) 13956 return SDValue(); 13957 // If we are converting to 32-bit integers, we need to add an FP_ROUND. 13958 // This is not valid if the input was originally double precision. It is 13959 // also not profitable to do unless this is an extending load in which 13960 // case doing this combine will allow us to combine consecutive loads. 13961 if (Is32Bit && !isFPExtLoad(NextOp.getOperand(0).getOperand(0))) 13962 return SDValue(); 13963 if (N->getOperand(i) != FirstInput) 13964 IsSplat = false; 13965 } 13966 13967 // If this is a splat, we leave it as-is since there will be only a single 13968 // fp-to-int conversion followed by a splat of the integer. This is better 13969 // for 32-bit and smaller ints and neutral for 64-bit ints. 13970 if (IsSplat) 13971 return SDValue(); 13972 13973 // Now that we know we have the right type of node, get its operands 13974 for (int i = 0, e = N->getNumOperands(); i < e; ++i) { 13975 SDValue In = N->getOperand(i).getOperand(0); 13976 if (Is32Bit) { 13977 // For 32-bit values, we need to add an FP_ROUND node (if we made it 13978 // here, we know that all inputs are extending loads so this is safe). 13979 if (In.isUndef()) 13980 Ops.push_back(DAG.getUNDEF(SrcVT)); 13981 else { 13982 SDValue Trunc = DAG.getNode(ISD::FP_ROUND, dl, 13983 MVT::f32, In.getOperand(0), 13984 DAG.getIntPtrConstant(1, dl)); 13985 Ops.push_back(Trunc); 13986 } 13987 } else 13988 Ops.push_back(In.isUndef() ? DAG.getUNDEF(SrcVT) : In.getOperand(0)); 13989 } 13990 13991 unsigned Opcode; 13992 if (FirstConversion == PPCISD::FCTIDZ || 13993 FirstConversion == PPCISD::FCTIWZ) 13994 Opcode = ISD::FP_TO_SINT; 13995 else 13996 Opcode = ISD::FP_TO_UINT; 13997 13998 EVT NewVT = TargetVT == MVT::v2i64 ? MVT::v2f64 : MVT::v4f32; 13999 SDValue BV = DAG.getBuildVector(NewVT, dl, Ops); 14000 return DAG.getNode(Opcode, dl, TargetVT, BV); 14001 } 14002 return SDValue(); 14003 } 14004 14005 /// Reduce the number of loads when building a vector. 14006 /// 14007 /// Building a vector out of multiple loads can be converted to a load 14008 /// of the vector type if the loads are consecutive. If the loads are 14009 /// consecutive but in descending order, a shuffle is added at the end 14010 /// to reorder the vector. 14011 static SDValue combineBVOfConsecutiveLoads(SDNode *N, SelectionDAG &DAG) { 14012 assert(N->getOpcode() == ISD::BUILD_VECTOR && 14013 "Should be called with a BUILD_VECTOR node"); 14014 14015 SDLoc dl(N); 14016 14017 // Return early for non byte-sized type, as they can't be consecutive. 14018 if (!N->getValueType(0).getVectorElementType().isByteSized()) 14019 return SDValue(); 14020 14021 bool InputsAreConsecutiveLoads = true; 14022 bool InputsAreReverseConsecutive = true; 14023 unsigned ElemSize = N->getValueType(0).getScalarType().getStoreSize(); 14024 SDValue FirstInput = N->getOperand(0); 14025 bool IsRoundOfExtLoad = false; 14026 14027 if (FirstInput.getOpcode() == ISD::FP_ROUND && 14028 FirstInput.getOperand(0).getOpcode() == ISD::LOAD) { 14029 LoadSDNode *LD = dyn_cast<LoadSDNode>(FirstInput.getOperand(0)); 14030 IsRoundOfExtLoad = LD->getExtensionType() == ISD::EXTLOAD; 14031 } 14032 // Not a build vector of (possibly fp_rounded) loads. 14033 if ((!IsRoundOfExtLoad && FirstInput.getOpcode() != ISD::LOAD) || 14034 N->getNumOperands() == 1) 14035 return SDValue(); 14036 14037 for (int i = 1, e = N->getNumOperands(); i < e; ++i) { 14038 // If any inputs are fp_round(extload), they all must be. 14039 if (IsRoundOfExtLoad && N->getOperand(i).getOpcode() != ISD::FP_ROUND) 14040 return SDValue(); 14041 14042 SDValue NextInput = IsRoundOfExtLoad ? N->getOperand(i).getOperand(0) : 14043 N->getOperand(i); 14044 if (NextInput.getOpcode() != ISD::LOAD) 14045 return SDValue(); 14046 14047 SDValue PreviousInput = 14048 IsRoundOfExtLoad ? N->getOperand(i-1).getOperand(0) : N->getOperand(i-1); 14049 LoadSDNode *LD1 = dyn_cast<LoadSDNode>(PreviousInput); 14050 LoadSDNode *LD2 = dyn_cast<LoadSDNode>(NextInput); 14051 14052 // If any inputs are fp_round(extload), they all must be. 14053 if (IsRoundOfExtLoad && LD2->getExtensionType() != ISD::EXTLOAD) 14054 return SDValue(); 14055 14056 if (!isConsecutiveLS(LD2, LD1, ElemSize, 1, DAG)) 14057 InputsAreConsecutiveLoads = false; 14058 if (!isConsecutiveLS(LD1, LD2, ElemSize, 1, DAG)) 14059 InputsAreReverseConsecutive = false; 14060 14061 // Exit early if the loads are neither consecutive nor reverse consecutive. 14062 if (!InputsAreConsecutiveLoads && !InputsAreReverseConsecutive) 14063 return SDValue(); 14064 } 14065 14066 assert(!(InputsAreConsecutiveLoads && InputsAreReverseConsecutive) && 14067 "The loads cannot be both consecutive and reverse consecutive."); 14068 14069 SDValue FirstLoadOp = 14070 IsRoundOfExtLoad ? FirstInput.getOperand(0) : FirstInput; 14071 SDValue LastLoadOp = 14072 IsRoundOfExtLoad ? N->getOperand(N->getNumOperands()-1).getOperand(0) : 14073 N->getOperand(N->getNumOperands()-1); 14074 14075 LoadSDNode *LD1 = dyn_cast<LoadSDNode>(FirstLoadOp); 14076 LoadSDNode *LDL = dyn_cast<LoadSDNode>(LastLoadOp); 14077 if (InputsAreConsecutiveLoads) { 14078 assert(LD1 && "Input needs to be a LoadSDNode."); 14079 return DAG.getLoad(N->getValueType(0), dl, LD1->getChain(), 14080 LD1->getBasePtr(), LD1->getPointerInfo(), 14081 LD1->getAlignment()); 14082 } 14083 if (InputsAreReverseConsecutive) { 14084 assert(LDL && "Input needs to be a LoadSDNode."); 14085 SDValue Load = DAG.getLoad(N->getValueType(0), dl, LDL->getChain(), 14086 LDL->getBasePtr(), LDL->getPointerInfo(), 14087 LDL->getAlignment()); 14088 SmallVector<int, 16> Ops; 14089 for (int i = N->getNumOperands() - 1; i >= 0; i--) 14090 Ops.push_back(i); 14091 14092 return DAG.getVectorShuffle(N->getValueType(0), dl, Load, 14093 DAG.getUNDEF(N->getValueType(0)), Ops); 14094 } 14095 return SDValue(); 14096 } 14097 14098 // This function adds the required vector_shuffle needed to get 14099 // the elements of the vector extract in the correct position 14100 // as specified by the CorrectElems encoding. 14101 static SDValue addShuffleForVecExtend(SDNode *N, SelectionDAG &DAG, 14102 SDValue Input, uint64_t Elems, 14103 uint64_t CorrectElems) { 14104 SDLoc dl(N); 14105 14106 unsigned NumElems = Input.getValueType().getVectorNumElements(); 14107 SmallVector<int, 16> ShuffleMask(NumElems, -1); 14108 14109 // Knowing the element indices being extracted from the original 14110 // vector and the order in which they're being inserted, just put 14111 // them at element indices required for the instruction. 14112 for (unsigned i = 0; i < N->getNumOperands(); i++) { 14113 if (DAG.getDataLayout().isLittleEndian()) 14114 ShuffleMask[CorrectElems & 0xF] = Elems & 0xF; 14115 else 14116 ShuffleMask[(CorrectElems & 0xF0) >> 4] = (Elems & 0xF0) >> 4; 14117 CorrectElems = CorrectElems >> 8; 14118 Elems = Elems >> 8; 14119 } 14120 14121 SDValue Shuffle = 14122 DAG.getVectorShuffle(Input.getValueType(), dl, Input, 14123 DAG.getUNDEF(Input.getValueType()), ShuffleMask); 14124 14125 EVT VT = N->getValueType(0); 14126 SDValue Conv = DAG.getBitcast(VT, Shuffle); 14127 14128 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), 14129 Input.getValueType().getVectorElementType(), 14130 VT.getVectorNumElements()); 14131 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, VT, Conv, 14132 DAG.getValueType(ExtVT)); 14133 } 14134 14135 // Look for build vector patterns where input operands come from sign 14136 // extended vector_extract elements of specific indices. If the correct indices 14137 // aren't used, add a vector shuffle to fix up the indices and create 14138 // SIGN_EXTEND_INREG node which selects the vector sign extend instructions 14139 // during instruction selection. 14140 static SDValue combineBVOfVecSExt(SDNode *N, SelectionDAG &DAG) { 14141 // This array encodes the indices that the vector sign extend instructions 14142 // extract from when extending from one type to another for both BE and LE. 14143 // The right nibble of each byte corresponds to the LE incides. 14144 // and the left nibble of each byte corresponds to the BE incides. 14145 // For example: 0x3074B8FC byte->word 14146 // For LE: the allowed indices are: 0x0,0x4,0x8,0xC 14147 // For BE: the allowed indices are: 0x3,0x7,0xB,0xF 14148 // For example: 0x000070F8 byte->double word 14149 // For LE: the allowed indices are: 0x0,0x8 14150 // For BE: the allowed indices are: 0x7,0xF 14151 uint64_t TargetElems[] = { 14152 0x3074B8FC, // b->w 14153 0x000070F8, // b->d 14154 0x10325476, // h->w 14155 0x00003074, // h->d 14156 0x00001032, // w->d 14157 }; 14158 14159 uint64_t Elems = 0; 14160 int Index; 14161 SDValue Input; 14162 14163 auto isSExtOfVecExtract = [&](SDValue Op) -> bool { 14164 if (!Op) 14165 return false; 14166 if (Op.getOpcode() != ISD::SIGN_EXTEND && 14167 Op.getOpcode() != ISD::SIGN_EXTEND_INREG) 14168 return false; 14169 14170 // A SIGN_EXTEND_INREG might be fed by an ANY_EXTEND to produce a value 14171 // of the right width. 14172 SDValue Extract = Op.getOperand(0); 14173 if (Extract.getOpcode() == ISD::ANY_EXTEND) 14174 Extract = Extract.getOperand(0); 14175 if (Extract.getOpcode() != ISD::EXTRACT_VECTOR_ELT) 14176 return false; 14177 14178 ConstantSDNode *ExtOp = dyn_cast<ConstantSDNode>(Extract.getOperand(1)); 14179 if (!ExtOp) 14180 return false; 14181 14182 Index = ExtOp->getZExtValue(); 14183 if (Input && Input != Extract.getOperand(0)) 14184 return false; 14185 14186 if (!Input) 14187 Input = Extract.getOperand(0); 14188 14189 Elems = Elems << 8; 14190 Index = DAG.getDataLayout().isLittleEndian() ? Index : Index << 4; 14191 Elems |= Index; 14192 14193 return true; 14194 }; 14195 14196 // If the build vector operands aren't sign extended vector extracts, 14197 // of the same input vector, then return. 14198 for (unsigned i = 0; i < N->getNumOperands(); i++) { 14199 if (!isSExtOfVecExtract(N->getOperand(i))) { 14200 return SDValue(); 14201 } 14202 } 14203 14204 // If the vector extract indicies are not correct, add the appropriate 14205 // vector_shuffle. 14206 int TgtElemArrayIdx; 14207 int InputSize = Input.getValueType().getScalarSizeInBits(); 14208 int OutputSize = N->getValueType(0).getScalarSizeInBits(); 14209 if (InputSize + OutputSize == 40) 14210 TgtElemArrayIdx = 0; 14211 else if (InputSize + OutputSize == 72) 14212 TgtElemArrayIdx = 1; 14213 else if (InputSize + OutputSize == 48) 14214 TgtElemArrayIdx = 2; 14215 else if (InputSize + OutputSize == 80) 14216 TgtElemArrayIdx = 3; 14217 else if (InputSize + OutputSize == 96) 14218 TgtElemArrayIdx = 4; 14219 else 14220 return SDValue(); 14221 14222 uint64_t CorrectElems = TargetElems[TgtElemArrayIdx]; 14223 CorrectElems = DAG.getDataLayout().isLittleEndian() 14224 ? CorrectElems & 0x0F0F0F0F0F0F0F0F 14225 : CorrectElems & 0xF0F0F0F0F0F0F0F0; 14226 if (Elems != CorrectElems) { 14227 return addShuffleForVecExtend(N, DAG, Input, Elems, CorrectElems); 14228 } 14229 14230 // Regular lowering will catch cases where a shuffle is not needed. 14231 return SDValue(); 14232 } 14233 14234 // Look for the pattern of a load from a narrow width to i128, feeding 14235 // into a BUILD_VECTOR of v1i128. Replace this sequence with a PPCISD node 14236 // (LXVRZX). This node represents a zero extending load that will be matched 14237 // to the Load VSX Vector Rightmost instructions. 14238 static SDValue combineBVZEXTLOAD(SDNode *N, SelectionDAG &DAG) { 14239 SDLoc DL(N); 14240 14241 // This combine is only eligible for a BUILD_VECTOR of v1i128. 14242 if (N->getValueType(0) != MVT::v1i128) 14243 return SDValue(); 14244 14245 SDValue Operand = N->getOperand(0); 14246 // Proceed with the transformation if the operand to the BUILD_VECTOR 14247 // is a load instruction. 14248 if (Operand.getOpcode() != ISD::LOAD) 14249 return SDValue(); 14250 14251 auto *LD = cast<LoadSDNode>(Operand); 14252 EVT MemoryType = LD->getMemoryVT(); 14253 14254 // This transformation is only valid if the we are loading either a byte, 14255 // halfword, word, or doubleword. 14256 bool ValidLDType = MemoryType == MVT::i8 || MemoryType == MVT::i16 || 14257 MemoryType == MVT::i32 || MemoryType == MVT::i64; 14258 14259 // Ensure that the load from the narrow width is being zero extended to i128. 14260 if (!ValidLDType || 14261 (LD->getExtensionType() != ISD::ZEXTLOAD && 14262 LD->getExtensionType() != ISD::EXTLOAD)) 14263 return SDValue(); 14264 14265 SDValue LoadOps[] = { 14266 LD->getChain(), LD->getBasePtr(), 14267 DAG.getIntPtrConstant(MemoryType.getScalarSizeInBits(), DL)}; 14268 14269 return DAG.getMemIntrinsicNode(PPCISD::LXVRZX, DL, 14270 DAG.getVTList(MVT::v1i128, MVT::Other), 14271 LoadOps, MemoryType, LD->getMemOperand()); 14272 } 14273 14274 SDValue PPCTargetLowering::DAGCombineBuildVector(SDNode *N, 14275 DAGCombinerInfo &DCI) const { 14276 assert(N->getOpcode() == ISD::BUILD_VECTOR && 14277 "Should be called with a BUILD_VECTOR node"); 14278 14279 SelectionDAG &DAG = DCI.DAG; 14280 SDLoc dl(N); 14281 14282 if (!Subtarget.hasVSX()) 14283 return SDValue(); 14284 14285 // The target independent DAG combiner will leave a build_vector of 14286 // float-to-int conversions intact. We can generate MUCH better code for 14287 // a float-to-int conversion of a vector of floats. 14288 SDValue FirstInput = N->getOperand(0); 14289 if (FirstInput.getOpcode() == PPCISD::MFVSR) { 14290 SDValue Reduced = combineElementTruncationToVectorTruncation(N, DCI); 14291 if (Reduced) 14292 return Reduced; 14293 } 14294 14295 // If we're building a vector out of consecutive loads, just load that 14296 // vector type. 14297 SDValue Reduced = combineBVOfConsecutiveLoads(N, DAG); 14298 if (Reduced) 14299 return Reduced; 14300 14301 // If we're building a vector out of extended elements from another vector 14302 // we have P9 vector integer extend instructions. The code assumes legal 14303 // input types (i.e. it can't handle things like v4i16) so do not run before 14304 // legalization. 14305 if (Subtarget.hasP9Altivec() && !DCI.isBeforeLegalize()) { 14306 Reduced = combineBVOfVecSExt(N, DAG); 14307 if (Reduced) 14308 return Reduced; 14309 } 14310 14311 // On Power10, the Load VSX Vector Rightmost instructions can be utilized 14312 // if this is a BUILD_VECTOR of v1i128, and if the operand to the BUILD_VECTOR 14313 // is a load from <valid narrow width> to i128. 14314 if (Subtarget.isISA3_1()) { 14315 SDValue BVOfZLoad = combineBVZEXTLOAD(N, DAG); 14316 if (BVOfZLoad) 14317 return BVOfZLoad; 14318 } 14319 14320 if (N->getValueType(0) != MVT::v2f64) 14321 return SDValue(); 14322 14323 // Looking for: 14324 // (build_vector ([su]int_to_fp (extractelt 0)), [su]int_to_fp (extractelt 1)) 14325 if (FirstInput.getOpcode() != ISD::SINT_TO_FP && 14326 FirstInput.getOpcode() != ISD::UINT_TO_FP) 14327 return SDValue(); 14328 if (N->getOperand(1).getOpcode() != ISD::SINT_TO_FP && 14329 N->getOperand(1).getOpcode() != ISD::UINT_TO_FP) 14330 return SDValue(); 14331 if (FirstInput.getOpcode() != N->getOperand(1).getOpcode()) 14332 return SDValue(); 14333 14334 SDValue Ext1 = FirstInput.getOperand(0); 14335 SDValue Ext2 = N->getOperand(1).getOperand(0); 14336 if(Ext1.getOpcode() != ISD::EXTRACT_VECTOR_ELT || 14337 Ext2.getOpcode() != ISD::EXTRACT_VECTOR_ELT) 14338 return SDValue(); 14339 14340 ConstantSDNode *Ext1Op = dyn_cast<ConstantSDNode>(Ext1.getOperand(1)); 14341 ConstantSDNode *Ext2Op = dyn_cast<ConstantSDNode>(Ext2.getOperand(1)); 14342 if (!Ext1Op || !Ext2Op) 14343 return SDValue(); 14344 if (Ext1.getOperand(0).getValueType() != MVT::v4i32 || 14345 Ext1.getOperand(0) != Ext2.getOperand(0)) 14346 return SDValue(); 14347 14348 int FirstElem = Ext1Op->getZExtValue(); 14349 int SecondElem = Ext2Op->getZExtValue(); 14350 int SubvecIdx; 14351 if (FirstElem == 0 && SecondElem == 1) 14352 SubvecIdx = Subtarget.isLittleEndian() ? 1 : 0; 14353 else if (FirstElem == 2 && SecondElem == 3) 14354 SubvecIdx = Subtarget.isLittleEndian() ? 0 : 1; 14355 else 14356 return SDValue(); 14357 14358 SDValue SrcVec = Ext1.getOperand(0); 14359 auto NodeType = (N->getOperand(1).getOpcode() == ISD::SINT_TO_FP) ? 14360 PPCISD::SINT_VEC_TO_FP : PPCISD::UINT_VEC_TO_FP; 14361 return DAG.getNode(NodeType, dl, MVT::v2f64, 14362 SrcVec, DAG.getIntPtrConstant(SubvecIdx, dl)); 14363 } 14364 14365 SDValue PPCTargetLowering::combineFPToIntToFP(SDNode *N, 14366 DAGCombinerInfo &DCI) const { 14367 assert((N->getOpcode() == ISD::SINT_TO_FP || 14368 N->getOpcode() == ISD::UINT_TO_FP) && 14369 "Need an int -> FP conversion node here"); 14370 14371 if (useSoftFloat() || !Subtarget.has64BitSupport()) 14372 return SDValue(); 14373 14374 SelectionDAG &DAG = DCI.DAG; 14375 SDLoc dl(N); 14376 SDValue Op(N, 0); 14377 14378 // Don't handle ppc_fp128 here or conversions that are out-of-range capable 14379 // from the hardware. 14380 if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64) 14381 return SDValue(); 14382 if (!Op.getOperand(0).getValueType().isSimple()) 14383 return SDValue(); 14384 if (Op.getOperand(0).getValueType().getSimpleVT() <= MVT(MVT::i1) || 14385 Op.getOperand(0).getValueType().getSimpleVT() > MVT(MVT::i64)) 14386 return SDValue(); 14387 14388 SDValue FirstOperand(Op.getOperand(0)); 14389 bool SubWordLoad = FirstOperand.getOpcode() == ISD::LOAD && 14390 (FirstOperand.getValueType() == MVT::i8 || 14391 FirstOperand.getValueType() == MVT::i16); 14392 if (Subtarget.hasP9Vector() && Subtarget.hasP9Altivec() && SubWordLoad) { 14393 bool Signed = N->getOpcode() == ISD::SINT_TO_FP; 14394 bool DstDouble = Op.getValueType() == MVT::f64; 14395 unsigned ConvOp = Signed ? 14396 (DstDouble ? PPCISD::FCFID : PPCISD::FCFIDS) : 14397 (DstDouble ? PPCISD::FCFIDU : PPCISD::FCFIDUS); 14398 SDValue WidthConst = 14399 DAG.getIntPtrConstant(FirstOperand.getValueType() == MVT::i8 ? 1 : 2, 14400 dl, false); 14401 LoadSDNode *LDN = cast<LoadSDNode>(FirstOperand.getNode()); 14402 SDValue Ops[] = { LDN->getChain(), LDN->getBasePtr(), WidthConst }; 14403 SDValue Ld = DAG.getMemIntrinsicNode(PPCISD::LXSIZX, dl, 14404 DAG.getVTList(MVT::f64, MVT::Other), 14405 Ops, MVT::i8, LDN->getMemOperand()); 14406 14407 // For signed conversion, we need to sign-extend the value in the VSR 14408 if (Signed) { 14409 SDValue ExtOps[] = { Ld, WidthConst }; 14410 SDValue Ext = DAG.getNode(PPCISD::VEXTS, dl, MVT::f64, ExtOps); 14411 return DAG.getNode(ConvOp, dl, DstDouble ? MVT::f64 : MVT::f32, Ext); 14412 } else 14413 return DAG.getNode(ConvOp, dl, DstDouble ? MVT::f64 : MVT::f32, Ld); 14414 } 14415 14416 14417 // For i32 intermediate values, unfortunately, the conversion functions 14418 // leave the upper 32 bits of the value are undefined. Within the set of 14419 // scalar instructions, we have no method for zero- or sign-extending the 14420 // value. Thus, we cannot handle i32 intermediate values here. 14421 if (Op.getOperand(0).getValueType() == MVT::i32) 14422 return SDValue(); 14423 14424 assert((Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) && 14425 "UINT_TO_FP is supported only with FPCVT"); 14426 14427 // If we have FCFIDS, then use it when converting to single-precision. 14428 // Otherwise, convert to double-precision and then round. 14429 unsigned FCFOp = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) 14430 ? (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDUS 14431 : PPCISD::FCFIDS) 14432 : (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDU 14433 : PPCISD::FCFID); 14434 MVT FCFTy = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) 14435 ? MVT::f32 14436 : MVT::f64; 14437 14438 // If we're converting from a float, to an int, and back to a float again, 14439 // then we don't need the store/load pair at all. 14440 if ((Op.getOperand(0).getOpcode() == ISD::FP_TO_UINT && 14441 Subtarget.hasFPCVT()) || 14442 (Op.getOperand(0).getOpcode() == ISD::FP_TO_SINT)) { 14443 SDValue Src = Op.getOperand(0).getOperand(0); 14444 if (Src.getValueType() == MVT::f32) { 14445 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src); 14446 DCI.AddToWorklist(Src.getNode()); 14447 } else if (Src.getValueType() != MVT::f64) { 14448 // Make sure that we don't pick up a ppc_fp128 source value. 14449 return SDValue(); 14450 } 14451 14452 unsigned FCTOp = 14453 Op.getOperand(0).getOpcode() == ISD::FP_TO_SINT ? PPCISD::FCTIDZ : 14454 PPCISD::FCTIDUZ; 14455 14456 SDValue Tmp = DAG.getNode(FCTOp, dl, MVT::f64, Src); 14457 SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Tmp); 14458 14459 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) { 14460 FP = DAG.getNode(ISD::FP_ROUND, dl, 14461 MVT::f32, FP, DAG.getIntPtrConstant(0, dl)); 14462 DCI.AddToWorklist(FP.getNode()); 14463 } 14464 14465 return FP; 14466 } 14467 14468 return SDValue(); 14469 } 14470 14471 // expandVSXLoadForLE - Convert VSX loads (which may be intrinsics for 14472 // builtins) into loads with swaps. 14473 SDValue PPCTargetLowering::expandVSXLoadForLE(SDNode *N, 14474 DAGCombinerInfo &DCI) const { 14475 SelectionDAG &DAG = DCI.DAG; 14476 SDLoc dl(N); 14477 SDValue Chain; 14478 SDValue Base; 14479 MachineMemOperand *MMO; 14480 14481 switch (N->getOpcode()) { 14482 default: 14483 llvm_unreachable("Unexpected opcode for little endian VSX load"); 14484 case ISD::LOAD: { 14485 LoadSDNode *LD = cast<LoadSDNode>(N); 14486 Chain = LD->getChain(); 14487 Base = LD->getBasePtr(); 14488 MMO = LD->getMemOperand(); 14489 // If the MMO suggests this isn't a load of a full vector, leave 14490 // things alone. For a built-in, we have to make the change for 14491 // correctness, so if there is a size problem that will be a bug. 14492 if (MMO->getSize() < 16) 14493 return SDValue(); 14494 break; 14495 } 14496 case ISD::INTRINSIC_W_CHAIN: { 14497 MemIntrinsicSDNode *Intrin = cast<MemIntrinsicSDNode>(N); 14498 Chain = Intrin->getChain(); 14499 // Similarly to the store case below, Intrin->getBasePtr() doesn't get 14500 // us what we want. Get operand 2 instead. 14501 Base = Intrin->getOperand(2); 14502 MMO = Intrin->getMemOperand(); 14503 break; 14504 } 14505 } 14506 14507 MVT VecTy = N->getValueType(0).getSimpleVT(); 14508 14509 // Do not expand to PPCISD::LXVD2X + PPCISD::XXSWAPD when the load is 14510 // aligned and the type is a vector with elements up to 4 bytes 14511 if (Subtarget.needsSwapsForVSXMemOps() && MMO->getAlign() >= Align(16) && 14512 VecTy.getScalarSizeInBits() <= 32) { 14513 return SDValue(); 14514 } 14515 14516 SDValue LoadOps[] = { Chain, Base }; 14517 SDValue Load = DAG.getMemIntrinsicNode(PPCISD::LXVD2X, dl, 14518 DAG.getVTList(MVT::v2f64, MVT::Other), 14519 LoadOps, MVT::v2f64, MMO); 14520 14521 DCI.AddToWorklist(Load.getNode()); 14522 Chain = Load.getValue(1); 14523 SDValue Swap = DAG.getNode( 14524 PPCISD::XXSWAPD, dl, DAG.getVTList(MVT::v2f64, MVT::Other), Chain, Load); 14525 DCI.AddToWorklist(Swap.getNode()); 14526 14527 // Add a bitcast if the resulting load type doesn't match v2f64. 14528 if (VecTy != MVT::v2f64) { 14529 SDValue N = DAG.getNode(ISD::BITCAST, dl, VecTy, Swap); 14530 DCI.AddToWorklist(N.getNode()); 14531 // Package {bitcast value, swap's chain} to match Load's shape. 14532 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(VecTy, MVT::Other), 14533 N, Swap.getValue(1)); 14534 } 14535 14536 return Swap; 14537 } 14538 14539 // expandVSXStoreForLE - Convert VSX stores (which may be intrinsics for 14540 // builtins) into stores with swaps. 14541 SDValue PPCTargetLowering::expandVSXStoreForLE(SDNode *N, 14542 DAGCombinerInfo &DCI) const { 14543 SelectionDAG &DAG = DCI.DAG; 14544 SDLoc dl(N); 14545 SDValue Chain; 14546 SDValue Base; 14547 unsigned SrcOpnd; 14548 MachineMemOperand *MMO; 14549 14550 switch (N->getOpcode()) { 14551 default: 14552 llvm_unreachable("Unexpected opcode for little endian VSX store"); 14553 case ISD::STORE: { 14554 StoreSDNode *ST = cast<StoreSDNode>(N); 14555 Chain = ST->getChain(); 14556 Base = ST->getBasePtr(); 14557 MMO = ST->getMemOperand(); 14558 SrcOpnd = 1; 14559 // If the MMO suggests this isn't a store of a full vector, leave 14560 // things alone. For a built-in, we have to make the change for 14561 // correctness, so if there is a size problem that will be a bug. 14562 if (MMO->getSize() < 16) 14563 return SDValue(); 14564 break; 14565 } 14566 case ISD::INTRINSIC_VOID: { 14567 MemIntrinsicSDNode *Intrin = cast<MemIntrinsicSDNode>(N); 14568 Chain = Intrin->getChain(); 14569 // Intrin->getBasePtr() oddly does not get what we want. 14570 Base = Intrin->getOperand(3); 14571 MMO = Intrin->getMemOperand(); 14572 SrcOpnd = 2; 14573 break; 14574 } 14575 } 14576 14577 SDValue Src = N->getOperand(SrcOpnd); 14578 MVT VecTy = Src.getValueType().getSimpleVT(); 14579 14580 // Do not expand to PPCISD::XXSWAPD and PPCISD::STXVD2X when the load is 14581 // aligned and the type is a vector with elements up to 4 bytes 14582 if (Subtarget.needsSwapsForVSXMemOps() && MMO->getAlign() >= Align(16) && 14583 VecTy.getScalarSizeInBits() <= 32) { 14584 return SDValue(); 14585 } 14586 14587 // All stores are done as v2f64 and possible bit cast. 14588 if (VecTy != MVT::v2f64) { 14589 Src = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Src); 14590 DCI.AddToWorklist(Src.getNode()); 14591 } 14592 14593 SDValue Swap = DAG.getNode(PPCISD::XXSWAPD, dl, 14594 DAG.getVTList(MVT::v2f64, MVT::Other), Chain, Src); 14595 DCI.AddToWorklist(Swap.getNode()); 14596 Chain = Swap.getValue(1); 14597 SDValue StoreOps[] = { Chain, Swap, Base }; 14598 SDValue Store = DAG.getMemIntrinsicNode(PPCISD::STXVD2X, dl, 14599 DAG.getVTList(MVT::Other), 14600 StoreOps, VecTy, MMO); 14601 DCI.AddToWorklist(Store.getNode()); 14602 return Store; 14603 } 14604 14605 // Handle DAG combine for STORE (FP_TO_INT F). 14606 SDValue PPCTargetLowering::combineStoreFPToInt(SDNode *N, 14607 DAGCombinerInfo &DCI) const { 14608 14609 SelectionDAG &DAG = DCI.DAG; 14610 SDLoc dl(N); 14611 unsigned Opcode = N->getOperand(1).getOpcode(); 14612 14613 assert((Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT) 14614 && "Not a FP_TO_INT Instruction!"); 14615 14616 SDValue Val = N->getOperand(1).getOperand(0); 14617 EVT Op1VT = N->getOperand(1).getValueType(); 14618 EVT ResVT = Val.getValueType(); 14619 14620 if (!isTypeLegal(ResVT)) 14621 return SDValue(); 14622 14623 // Only perform combine for conversion to i64/i32 or power9 i16/i8. 14624 bool ValidTypeForStoreFltAsInt = 14625 (Op1VT == MVT::i32 || Op1VT == MVT::i64 || 14626 (Subtarget.hasP9Vector() && (Op1VT == MVT::i16 || Op1VT == MVT::i8))); 14627 14628 if (ResVT == MVT::f128 && !Subtarget.hasP9Vector()) 14629 return SDValue(); 14630 14631 if (ResVT == MVT::ppcf128 || !Subtarget.hasP8Vector() || 14632 cast<StoreSDNode>(N)->isTruncatingStore() || !ValidTypeForStoreFltAsInt) 14633 return SDValue(); 14634 14635 // Extend f32 values to f64 14636 if (ResVT.getScalarSizeInBits() == 32) { 14637 Val = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Val); 14638 DCI.AddToWorklist(Val.getNode()); 14639 } 14640 14641 // Set signed or unsigned conversion opcode. 14642 unsigned ConvOpcode = (Opcode == ISD::FP_TO_SINT) ? 14643 PPCISD::FP_TO_SINT_IN_VSR : 14644 PPCISD::FP_TO_UINT_IN_VSR; 14645 14646 Val = DAG.getNode(ConvOpcode, 14647 dl, ResVT == MVT::f128 ? MVT::f128 : MVT::f64, Val); 14648 DCI.AddToWorklist(Val.getNode()); 14649 14650 // Set number of bytes being converted. 14651 unsigned ByteSize = Op1VT.getScalarSizeInBits() / 8; 14652 SDValue Ops[] = { N->getOperand(0), Val, N->getOperand(2), 14653 DAG.getIntPtrConstant(ByteSize, dl, false), 14654 DAG.getValueType(Op1VT) }; 14655 14656 Val = DAG.getMemIntrinsicNode(PPCISD::ST_VSR_SCAL_INT, dl, 14657 DAG.getVTList(MVT::Other), Ops, 14658 cast<StoreSDNode>(N)->getMemoryVT(), 14659 cast<StoreSDNode>(N)->getMemOperand()); 14660 14661 DCI.AddToWorklist(Val.getNode()); 14662 return Val; 14663 } 14664 14665 static bool isAlternatingShuffMask(const ArrayRef<int> &Mask, int NumElts) { 14666 // Check that the source of the element keeps flipping 14667 // (i.e. Mask[i] < NumElts -> Mask[i+i] >= NumElts). 14668 bool PrevElemFromFirstVec = Mask[0] < NumElts; 14669 for (int i = 1, e = Mask.size(); i < e; i++) { 14670 if (PrevElemFromFirstVec && Mask[i] < NumElts) 14671 return false; 14672 if (!PrevElemFromFirstVec && Mask[i] >= NumElts) 14673 return false; 14674 PrevElemFromFirstVec = !PrevElemFromFirstVec; 14675 } 14676 return true; 14677 } 14678 14679 static bool isSplatBV(SDValue Op) { 14680 if (Op.getOpcode() != ISD::BUILD_VECTOR) 14681 return false; 14682 SDValue FirstOp; 14683 14684 // Find first non-undef input. 14685 for (int i = 0, e = Op.getNumOperands(); i < e; i++) { 14686 FirstOp = Op.getOperand(i); 14687 if (!FirstOp.isUndef()) 14688 break; 14689 } 14690 14691 // All inputs are undef or the same as the first non-undef input. 14692 for (int i = 1, e = Op.getNumOperands(); i < e; i++) 14693 if (Op.getOperand(i) != FirstOp && !Op.getOperand(i).isUndef()) 14694 return false; 14695 return true; 14696 } 14697 14698 static SDValue isScalarToVec(SDValue Op) { 14699 if (Op.getOpcode() == ISD::SCALAR_TO_VECTOR) 14700 return Op; 14701 if (Op.getOpcode() != ISD::BITCAST) 14702 return SDValue(); 14703 Op = Op.getOperand(0); 14704 if (Op.getOpcode() == ISD::SCALAR_TO_VECTOR) 14705 return Op; 14706 return SDValue(); 14707 } 14708 14709 // Fix up the shuffle mask to account for the fact that the result of 14710 // scalar_to_vector is not in lane zero. This just takes all values in 14711 // the ranges specified by the min/max indices and adds the number of 14712 // elements required to ensure each element comes from the respective 14713 // position in the valid lane. 14714 // On little endian, that's just the corresponding element in the other 14715 // half of the vector. On big endian, it is in the same half but right 14716 // justified rather than left justified in that half. 14717 static void fixupShuffleMaskForPermutedSToV(SmallVectorImpl<int> &ShuffV, 14718 int LHSMaxIdx, int RHSMinIdx, 14719 int RHSMaxIdx, int HalfVec, 14720 unsigned ValidLaneWidth, 14721 const PPCSubtarget &Subtarget) { 14722 for (int i = 0, e = ShuffV.size(); i < e; i++) { 14723 int Idx = ShuffV[i]; 14724 if ((Idx >= 0 && Idx < LHSMaxIdx) || (Idx >= RHSMinIdx && Idx < RHSMaxIdx)) 14725 ShuffV[i] += 14726 Subtarget.isLittleEndian() ? HalfVec : HalfVec - ValidLaneWidth; 14727 } 14728 } 14729 14730 // Replace a SCALAR_TO_VECTOR with a SCALAR_TO_VECTOR_PERMUTED except if 14731 // the original is: 14732 // (<n x Ty> (scalar_to_vector (Ty (extract_elt <n x Ty> %a, C)))) 14733 // In such a case, just change the shuffle mask to extract the element 14734 // from the permuted index. 14735 static SDValue getSToVPermuted(SDValue OrigSToV, SelectionDAG &DAG, 14736 const PPCSubtarget &Subtarget) { 14737 SDLoc dl(OrigSToV); 14738 EVT VT = OrigSToV.getValueType(); 14739 assert(OrigSToV.getOpcode() == ISD::SCALAR_TO_VECTOR && 14740 "Expecting a SCALAR_TO_VECTOR here"); 14741 SDValue Input = OrigSToV.getOperand(0); 14742 14743 if (Input.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { 14744 ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(Input.getOperand(1)); 14745 SDValue OrigVector = Input.getOperand(0); 14746 14747 // Can't handle non-const element indices or different vector types 14748 // for the input to the extract and the output of the scalar_to_vector. 14749 if (Idx && VT == OrigVector.getValueType()) { 14750 unsigned NumElts = VT.getVectorNumElements(); 14751 assert( 14752 NumElts > 1 && 14753 "Cannot produce a permuted scalar_to_vector for one element vector"); 14754 SmallVector<int, 16> NewMask(NumElts, -1); 14755 unsigned ResultInElt = NumElts / 2; 14756 ResultInElt -= Subtarget.isLittleEndian() ? 0 : 1; 14757 NewMask[ResultInElt] = Idx->getZExtValue(); 14758 return DAG.getVectorShuffle(VT, dl, OrigVector, OrigVector, NewMask); 14759 } 14760 } 14761 return DAG.getNode(PPCISD::SCALAR_TO_VECTOR_PERMUTED, dl, VT, 14762 OrigSToV.getOperand(0)); 14763 } 14764 14765 // On little endian subtargets, combine shuffles such as: 14766 // vector_shuffle<16,1,17,3,18,5,19,7,20,9,21,11,22,13,23,15>, <zero>, %b 14767 // into: 14768 // vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7>, <zero>, %b 14769 // because the latter can be matched to a single instruction merge. 14770 // Furthermore, SCALAR_TO_VECTOR on little endian always involves a permute 14771 // to put the value into element zero. Adjust the shuffle mask so that the 14772 // vector can remain in permuted form (to prevent a swap prior to a shuffle). 14773 // On big endian targets, this is still useful for SCALAR_TO_VECTOR 14774 // nodes with elements smaller than doubleword because all the ways 14775 // of getting scalar data into a vector register put the value in the 14776 // rightmost element of the left half of the vector. 14777 SDValue PPCTargetLowering::combineVectorShuffle(ShuffleVectorSDNode *SVN, 14778 SelectionDAG &DAG) const { 14779 SDValue LHS = SVN->getOperand(0); 14780 SDValue RHS = SVN->getOperand(1); 14781 auto Mask = SVN->getMask(); 14782 int NumElts = LHS.getValueType().getVectorNumElements(); 14783 SDValue Res(SVN, 0); 14784 SDLoc dl(SVN); 14785 bool IsLittleEndian = Subtarget.isLittleEndian(); 14786 14787 // On big endian targets this is only useful for subtargets with direct moves. 14788 // On little endian targets it would be useful for all subtargets with VSX. 14789 // However adding special handling for LE subtargets without direct moves 14790 // would be wasted effort since the minimum arch for LE is ISA 2.07 (Power8) 14791 // which includes direct moves. 14792 if (!Subtarget.hasDirectMove()) 14793 return Res; 14794 14795 // If this is not a shuffle of a shuffle and the first element comes from 14796 // the second vector, canonicalize to the commuted form. This will make it 14797 // more likely to match one of the single instruction patterns. 14798 if (Mask[0] >= NumElts && LHS.getOpcode() != ISD::VECTOR_SHUFFLE && 14799 RHS.getOpcode() != ISD::VECTOR_SHUFFLE) { 14800 std::swap(LHS, RHS); 14801 Res = DAG.getCommutedVectorShuffle(*SVN); 14802 Mask = cast<ShuffleVectorSDNode>(Res)->getMask(); 14803 } 14804 14805 // Adjust the shuffle mask if either input vector comes from a 14806 // SCALAR_TO_VECTOR and keep the respective input vector in permuted 14807 // form (to prevent the need for a swap). 14808 SmallVector<int, 16> ShuffV(Mask.begin(), Mask.end()); 14809 SDValue SToVLHS = isScalarToVec(LHS); 14810 SDValue SToVRHS = isScalarToVec(RHS); 14811 if (SToVLHS || SToVRHS) { 14812 int NumEltsIn = SToVLHS ? SToVLHS.getValueType().getVectorNumElements() 14813 : SToVRHS.getValueType().getVectorNumElements(); 14814 int NumEltsOut = ShuffV.size(); 14815 // The width of the "valid lane" (i.e. the lane that contains the value that 14816 // is vectorized) needs to be expressed in terms of the number of elements 14817 // of the shuffle. It is thereby the ratio of the values before and after 14818 // any bitcast. 14819 unsigned ValidLaneWidth = 14820 SToVLHS ? SToVLHS.getValueType().getScalarSizeInBits() / 14821 LHS.getValueType().getScalarSizeInBits() 14822 : SToVRHS.getValueType().getScalarSizeInBits() / 14823 RHS.getValueType().getScalarSizeInBits(); 14824 14825 // Initially assume that neither input is permuted. These will be adjusted 14826 // accordingly if either input is. 14827 int LHSMaxIdx = -1; 14828 int RHSMinIdx = -1; 14829 int RHSMaxIdx = -1; 14830 int HalfVec = LHS.getValueType().getVectorNumElements() / 2; 14831 14832 // Get the permuted scalar to vector nodes for the source(s) that come from 14833 // ISD::SCALAR_TO_VECTOR. 14834 // On big endian systems, this only makes sense for element sizes smaller 14835 // than 64 bits since for 64-bit elements, all instructions already put 14836 // the value into element zero. Since scalar size of LHS and RHS may differ 14837 // after isScalarToVec, this should be checked using their own sizes. 14838 if (SToVLHS) { 14839 if (!IsLittleEndian && SToVLHS.getValueType().getScalarSizeInBits() >= 64) 14840 return Res; 14841 // Set up the values for the shuffle vector fixup. 14842 LHSMaxIdx = NumEltsOut / NumEltsIn; 14843 SToVLHS = getSToVPermuted(SToVLHS, DAG, Subtarget); 14844 if (SToVLHS.getValueType() != LHS.getValueType()) 14845 SToVLHS = DAG.getBitcast(LHS.getValueType(), SToVLHS); 14846 LHS = SToVLHS; 14847 } 14848 if (SToVRHS) { 14849 if (!IsLittleEndian && SToVRHS.getValueType().getScalarSizeInBits() >= 64) 14850 return Res; 14851 RHSMinIdx = NumEltsOut; 14852 RHSMaxIdx = NumEltsOut / NumEltsIn + RHSMinIdx; 14853 SToVRHS = getSToVPermuted(SToVRHS, DAG, Subtarget); 14854 if (SToVRHS.getValueType() != RHS.getValueType()) 14855 SToVRHS = DAG.getBitcast(RHS.getValueType(), SToVRHS); 14856 RHS = SToVRHS; 14857 } 14858 14859 // Fix up the shuffle mask to reflect where the desired element actually is. 14860 // The minimum and maximum indices that correspond to element zero for both 14861 // the LHS and RHS are computed and will control which shuffle mask entries 14862 // are to be changed. For example, if the RHS is permuted, any shuffle mask 14863 // entries in the range [RHSMinIdx,RHSMaxIdx) will be adjusted. 14864 fixupShuffleMaskForPermutedSToV(ShuffV, LHSMaxIdx, RHSMinIdx, RHSMaxIdx, 14865 HalfVec, ValidLaneWidth, Subtarget); 14866 Res = DAG.getVectorShuffle(SVN->getValueType(0), dl, LHS, RHS, ShuffV); 14867 14868 // We may have simplified away the shuffle. We won't be able to do anything 14869 // further with it here. 14870 if (!isa<ShuffleVectorSDNode>(Res)) 14871 return Res; 14872 Mask = cast<ShuffleVectorSDNode>(Res)->getMask(); 14873 } 14874 14875 SDValue TheSplat = IsLittleEndian ? RHS : LHS; 14876 // The common case after we commuted the shuffle is that the RHS is a splat 14877 // and we have elements coming in from the splat at indices that are not 14878 // conducive to using a merge. 14879 // Example: 14880 // vector_shuffle<0,17,1,19,2,21,3,23,4,25,5,27,6,29,7,31> t1, <zero> 14881 if (!isSplatBV(TheSplat)) 14882 return Res; 14883 14884 // We are looking for a mask such that all even elements are from 14885 // one vector and all odd elements from the other. 14886 if (!isAlternatingShuffMask(Mask, NumElts)) 14887 return Res; 14888 14889 // Adjust the mask so we are pulling in the same index from the splat 14890 // as the index from the interesting vector in consecutive elements. 14891 if (IsLittleEndian) { 14892 // Example (even elements from first vector): 14893 // vector_shuffle<0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23> t1, <zero> 14894 if (Mask[0] < NumElts) 14895 for (int i = 1, e = Mask.size(); i < e; i += 2) 14896 ShuffV[i] = (ShuffV[i - 1] + NumElts); 14897 // Example (odd elements from first vector): 14898 // vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7> t1, <zero> 14899 else 14900 for (int i = 0, e = Mask.size(); i < e; i += 2) 14901 ShuffV[i] = (ShuffV[i + 1] + NumElts); 14902 } else { 14903 // Example (even elements from first vector): 14904 // vector_shuffle<0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23> <zero>, t1 14905 if (Mask[0] < NumElts) 14906 for (int i = 0, e = Mask.size(); i < e; i += 2) 14907 ShuffV[i] = ShuffV[i + 1] - NumElts; 14908 // Example (odd elements from first vector): 14909 // vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7> <zero>, t1 14910 else 14911 for (int i = 1, e = Mask.size(); i < e; i += 2) 14912 ShuffV[i] = ShuffV[i - 1] - NumElts; 14913 } 14914 14915 // If the RHS has undefs, we need to remove them since we may have created 14916 // a shuffle that adds those instead of the splat value. 14917 SDValue SplatVal = 14918 cast<BuildVectorSDNode>(TheSplat.getNode())->getSplatValue(); 14919 TheSplat = DAG.getSplatBuildVector(TheSplat.getValueType(), dl, SplatVal); 14920 14921 if (IsLittleEndian) 14922 RHS = TheSplat; 14923 else 14924 LHS = TheSplat; 14925 return DAG.getVectorShuffle(SVN->getValueType(0), dl, LHS, RHS, ShuffV); 14926 } 14927 14928 SDValue PPCTargetLowering::combineVReverseMemOP(ShuffleVectorSDNode *SVN, 14929 LSBaseSDNode *LSBase, 14930 DAGCombinerInfo &DCI) const { 14931 assert((ISD::isNormalLoad(LSBase) || ISD::isNormalStore(LSBase)) && 14932 "Not a reverse memop pattern!"); 14933 14934 auto IsElementReverse = [](const ShuffleVectorSDNode *SVN) -> bool { 14935 auto Mask = SVN->getMask(); 14936 int i = 0; 14937 auto I = Mask.rbegin(); 14938 auto E = Mask.rend(); 14939 14940 for (; I != E; ++I) { 14941 if (*I != i) 14942 return false; 14943 i++; 14944 } 14945 return true; 14946 }; 14947 14948 SelectionDAG &DAG = DCI.DAG; 14949 EVT VT = SVN->getValueType(0); 14950 14951 if (!isTypeLegal(VT) || !Subtarget.isLittleEndian() || !Subtarget.hasVSX()) 14952 return SDValue(); 14953 14954 // Before P9, we have PPCVSXSwapRemoval pass to hack the element order. 14955 // See comment in PPCVSXSwapRemoval.cpp. 14956 // It is conflict with PPCVSXSwapRemoval opt. So we don't do it. 14957 if (!Subtarget.hasP9Vector()) 14958 return SDValue(); 14959 14960 if(!IsElementReverse(SVN)) 14961 return SDValue(); 14962 14963 if (LSBase->getOpcode() == ISD::LOAD) { 14964 // If the load return value 0 has more than one user except the 14965 // shufflevector instruction, it is not profitable to replace the 14966 // shufflevector with a reverse load. 14967 for (SDNode::use_iterator UI = LSBase->use_begin(), UE = LSBase->use_end(); 14968 UI != UE; ++UI) 14969 if (UI.getUse().getResNo() == 0 && UI->getOpcode() != ISD::VECTOR_SHUFFLE) 14970 return SDValue(); 14971 14972 SDLoc dl(LSBase); 14973 SDValue LoadOps[] = {LSBase->getChain(), LSBase->getBasePtr()}; 14974 return DAG.getMemIntrinsicNode( 14975 PPCISD::LOAD_VEC_BE, dl, DAG.getVTList(VT, MVT::Other), LoadOps, 14976 LSBase->getMemoryVT(), LSBase->getMemOperand()); 14977 } 14978 14979 if (LSBase->getOpcode() == ISD::STORE) { 14980 // If there are other uses of the shuffle, the swap cannot be avoided. 14981 // Forcing the use of an X-Form (since swapped stores only have 14982 // X-Forms) without removing the swap is unprofitable. 14983 if (!SVN->hasOneUse()) 14984 return SDValue(); 14985 14986 SDLoc dl(LSBase); 14987 SDValue StoreOps[] = {LSBase->getChain(), SVN->getOperand(0), 14988 LSBase->getBasePtr()}; 14989 return DAG.getMemIntrinsicNode( 14990 PPCISD::STORE_VEC_BE, dl, DAG.getVTList(MVT::Other), StoreOps, 14991 LSBase->getMemoryVT(), LSBase->getMemOperand()); 14992 } 14993 14994 llvm_unreachable("Expected a load or store node here"); 14995 } 14996 14997 SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N, 14998 DAGCombinerInfo &DCI) const { 14999 SelectionDAG &DAG = DCI.DAG; 15000 SDLoc dl(N); 15001 switch (N->getOpcode()) { 15002 default: break; 15003 case ISD::ADD: 15004 return combineADD(N, DCI); 15005 case ISD::SHL: 15006 return combineSHL(N, DCI); 15007 case ISD::SRA: 15008 return combineSRA(N, DCI); 15009 case ISD::SRL: 15010 return combineSRL(N, DCI); 15011 case ISD::MUL: 15012 return combineMUL(N, DCI); 15013 case ISD::FMA: 15014 case PPCISD::FNMSUB: 15015 return combineFMALike(N, DCI); 15016 case PPCISD::SHL: 15017 if (isNullConstant(N->getOperand(0))) // 0 << V -> 0. 15018 return N->getOperand(0); 15019 break; 15020 case PPCISD::SRL: 15021 if (isNullConstant(N->getOperand(0))) // 0 >>u V -> 0. 15022 return N->getOperand(0); 15023 break; 15024 case PPCISD::SRA: 15025 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(0))) { 15026 if (C->isZero() || // 0 >>s V -> 0. 15027 C->isAllOnes()) // -1 >>s V -> -1. 15028 return N->getOperand(0); 15029 } 15030 break; 15031 case ISD::SIGN_EXTEND: 15032 case ISD::ZERO_EXTEND: 15033 case ISD::ANY_EXTEND: 15034 return DAGCombineExtBoolTrunc(N, DCI); 15035 case ISD::TRUNCATE: 15036 return combineTRUNCATE(N, DCI); 15037 case ISD::SETCC: 15038 if (SDValue CSCC = combineSetCC(N, DCI)) 15039 return CSCC; 15040 LLVM_FALLTHROUGH; 15041 case ISD::SELECT_CC: 15042 return DAGCombineTruncBoolExt(N, DCI); 15043 case ISD::SINT_TO_FP: 15044 case ISD::UINT_TO_FP: 15045 return combineFPToIntToFP(N, DCI); 15046 case ISD::VECTOR_SHUFFLE: 15047 if (ISD::isNormalLoad(N->getOperand(0).getNode())) { 15048 LSBaseSDNode* LSBase = cast<LSBaseSDNode>(N->getOperand(0)); 15049 return combineVReverseMemOP(cast<ShuffleVectorSDNode>(N), LSBase, DCI); 15050 } 15051 return combineVectorShuffle(cast<ShuffleVectorSDNode>(N), DCI.DAG); 15052 case ISD::STORE: { 15053 15054 EVT Op1VT = N->getOperand(1).getValueType(); 15055 unsigned Opcode = N->getOperand(1).getOpcode(); 15056 15057 if (Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT) { 15058 SDValue Val= combineStoreFPToInt(N, DCI); 15059 if (Val) 15060 return Val; 15061 } 15062 15063 if (Opcode == ISD::VECTOR_SHUFFLE && ISD::isNormalStore(N)) { 15064 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N->getOperand(1)); 15065 SDValue Val= combineVReverseMemOP(SVN, cast<LSBaseSDNode>(N), DCI); 15066 if (Val) 15067 return Val; 15068 } 15069 15070 // Turn STORE (BSWAP) -> sthbrx/stwbrx. 15071 if (cast<StoreSDNode>(N)->isUnindexed() && Opcode == ISD::BSWAP && 15072 N->getOperand(1).getNode()->hasOneUse() && 15073 (Op1VT == MVT::i32 || Op1VT == MVT::i16 || 15074 (Subtarget.hasLDBRX() && Subtarget.isPPC64() && Op1VT == MVT::i64))) { 15075 15076 // STBRX can only handle simple types and it makes no sense to store less 15077 // two bytes in byte-reversed order. 15078 EVT mVT = cast<StoreSDNode>(N)->getMemoryVT(); 15079 if (mVT.isExtended() || mVT.getSizeInBits() < 16) 15080 break; 15081 15082 SDValue BSwapOp = N->getOperand(1).getOperand(0); 15083 // Do an any-extend to 32-bits if this is a half-word input. 15084 if (BSwapOp.getValueType() == MVT::i16) 15085 BSwapOp = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, BSwapOp); 15086 15087 // If the type of BSWAP operand is wider than stored memory width 15088 // it need to be shifted to the right side before STBRX. 15089 if (Op1VT.bitsGT(mVT)) { 15090 int Shift = Op1VT.getSizeInBits() - mVT.getSizeInBits(); 15091 BSwapOp = DAG.getNode(ISD::SRL, dl, Op1VT, BSwapOp, 15092 DAG.getConstant(Shift, dl, MVT::i32)); 15093 // Need to truncate if this is a bswap of i64 stored as i32/i16. 15094 if (Op1VT == MVT::i64) 15095 BSwapOp = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, BSwapOp); 15096 } 15097 15098 SDValue Ops[] = { 15099 N->getOperand(0), BSwapOp, N->getOperand(2), DAG.getValueType(mVT) 15100 }; 15101 return 15102 DAG.getMemIntrinsicNode(PPCISD::STBRX, dl, DAG.getVTList(MVT::Other), 15103 Ops, cast<StoreSDNode>(N)->getMemoryVT(), 15104 cast<StoreSDNode>(N)->getMemOperand()); 15105 } 15106 15107 // STORE Constant:i32<0> -> STORE<trunc to i32> Constant:i64<0> 15108 // So it can increase the chance of CSE constant construction. 15109 if (Subtarget.isPPC64() && !DCI.isBeforeLegalize() && 15110 isa<ConstantSDNode>(N->getOperand(1)) && Op1VT == MVT::i32) { 15111 // Need to sign-extended to 64-bits to handle negative values. 15112 EVT MemVT = cast<StoreSDNode>(N)->getMemoryVT(); 15113 uint64_t Val64 = SignExtend64(N->getConstantOperandVal(1), 15114 MemVT.getSizeInBits()); 15115 SDValue Const64 = DAG.getConstant(Val64, dl, MVT::i64); 15116 15117 // DAG.getTruncStore() can't be used here because it doesn't accept 15118 // the general (base + offset) addressing mode. 15119 // So we use UpdateNodeOperands and setTruncatingStore instead. 15120 DAG.UpdateNodeOperands(N, N->getOperand(0), Const64, N->getOperand(2), 15121 N->getOperand(3)); 15122 cast<StoreSDNode>(N)->setTruncatingStore(true); 15123 return SDValue(N, 0); 15124 } 15125 15126 // For little endian, VSX stores require generating xxswapd/lxvd2x. 15127 // Not needed on ISA 3.0 based CPUs since we have a non-permuting store. 15128 if (Op1VT.isSimple()) { 15129 MVT StoreVT = Op1VT.getSimpleVT(); 15130 if (Subtarget.needsSwapsForVSXMemOps() && 15131 (StoreVT == MVT::v2f64 || StoreVT == MVT::v2i64 || 15132 StoreVT == MVT::v4f32 || StoreVT == MVT::v4i32)) 15133 return expandVSXStoreForLE(N, DCI); 15134 } 15135 break; 15136 } 15137 case ISD::LOAD: { 15138 LoadSDNode *LD = cast<LoadSDNode>(N); 15139 EVT VT = LD->getValueType(0); 15140 15141 // For little endian, VSX loads require generating lxvd2x/xxswapd. 15142 // Not needed on ISA 3.0 based CPUs since we have a non-permuting load. 15143 if (VT.isSimple()) { 15144 MVT LoadVT = VT.getSimpleVT(); 15145 if (Subtarget.needsSwapsForVSXMemOps() && 15146 (LoadVT == MVT::v2f64 || LoadVT == MVT::v2i64 || 15147 LoadVT == MVT::v4f32 || LoadVT == MVT::v4i32)) 15148 return expandVSXLoadForLE(N, DCI); 15149 } 15150 15151 // We sometimes end up with a 64-bit integer load, from which we extract 15152 // two single-precision floating-point numbers. This happens with 15153 // std::complex<float>, and other similar structures, because of the way we 15154 // canonicalize structure copies. However, if we lack direct moves, 15155 // then the final bitcasts from the extracted integer values to the 15156 // floating-point numbers turn into store/load pairs. Even with direct moves, 15157 // just loading the two floating-point numbers is likely better. 15158 auto ReplaceTwoFloatLoad = [&]() { 15159 if (VT != MVT::i64) 15160 return false; 15161 15162 if (LD->getExtensionType() != ISD::NON_EXTLOAD || 15163 LD->isVolatile()) 15164 return false; 15165 15166 // We're looking for a sequence like this: 15167 // t13: i64,ch = load<LD8[%ref.tmp]> t0, t6, undef:i64 15168 // t16: i64 = srl t13, Constant:i32<32> 15169 // t17: i32 = truncate t16 15170 // t18: f32 = bitcast t17 15171 // t19: i32 = truncate t13 15172 // t20: f32 = bitcast t19 15173 15174 if (!LD->hasNUsesOfValue(2, 0)) 15175 return false; 15176 15177 auto UI = LD->use_begin(); 15178 while (UI.getUse().getResNo() != 0) ++UI; 15179 SDNode *Trunc = *UI++; 15180 while (UI.getUse().getResNo() != 0) ++UI; 15181 SDNode *RightShift = *UI; 15182 if (Trunc->getOpcode() != ISD::TRUNCATE) 15183 std::swap(Trunc, RightShift); 15184 15185 if (Trunc->getOpcode() != ISD::TRUNCATE || 15186 Trunc->getValueType(0) != MVT::i32 || 15187 !Trunc->hasOneUse()) 15188 return false; 15189 if (RightShift->getOpcode() != ISD::SRL || 15190 !isa<ConstantSDNode>(RightShift->getOperand(1)) || 15191 RightShift->getConstantOperandVal(1) != 32 || 15192 !RightShift->hasOneUse()) 15193 return false; 15194 15195 SDNode *Trunc2 = *RightShift->use_begin(); 15196 if (Trunc2->getOpcode() != ISD::TRUNCATE || 15197 Trunc2->getValueType(0) != MVT::i32 || 15198 !Trunc2->hasOneUse()) 15199 return false; 15200 15201 SDNode *Bitcast = *Trunc->use_begin(); 15202 SDNode *Bitcast2 = *Trunc2->use_begin(); 15203 15204 if (Bitcast->getOpcode() != ISD::BITCAST || 15205 Bitcast->getValueType(0) != MVT::f32) 15206 return false; 15207 if (Bitcast2->getOpcode() != ISD::BITCAST || 15208 Bitcast2->getValueType(0) != MVT::f32) 15209 return false; 15210 15211 if (Subtarget.isLittleEndian()) 15212 std::swap(Bitcast, Bitcast2); 15213 15214 // Bitcast has the second float (in memory-layout order) and Bitcast2 15215 // has the first one. 15216 15217 SDValue BasePtr = LD->getBasePtr(); 15218 if (LD->isIndexed()) { 15219 assert(LD->getAddressingMode() == ISD::PRE_INC && 15220 "Non-pre-inc AM on PPC?"); 15221 BasePtr = 15222 DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 15223 LD->getOffset()); 15224 } 15225 15226 auto MMOFlags = 15227 LD->getMemOperand()->getFlags() & ~MachineMemOperand::MOVolatile; 15228 SDValue FloatLoad = DAG.getLoad(MVT::f32, dl, LD->getChain(), BasePtr, 15229 LD->getPointerInfo(), LD->getAlignment(), 15230 MMOFlags, LD->getAAInfo()); 15231 SDValue AddPtr = 15232 DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 15233 BasePtr, DAG.getIntPtrConstant(4, dl)); 15234 SDValue FloatLoad2 = DAG.getLoad( 15235 MVT::f32, dl, SDValue(FloatLoad.getNode(), 1), AddPtr, 15236 LD->getPointerInfo().getWithOffset(4), 15237 MinAlign(LD->getAlignment(), 4), MMOFlags, LD->getAAInfo()); 15238 15239 if (LD->isIndexed()) { 15240 // Note that DAGCombine should re-form any pre-increment load(s) from 15241 // what is produced here if that makes sense. 15242 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), BasePtr); 15243 } 15244 15245 DCI.CombineTo(Bitcast2, FloatLoad); 15246 DCI.CombineTo(Bitcast, FloatLoad2); 15247 15248 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, LD->isIndexed() ? 2 : 1), 15249 SDValue(FloatLoad2.getNode(), 1)); 15250 return true; 15251 }; 15252 15253 if (ReplaceTwoFloatLoad()) 15254 return SDValue(N, 0); 15255 15256 EVT MemVT = LD->getMemoryVT(); 15257 Type *Ty = MemVT.getTypeForEVT(*DAG.getContext()); 15258 Align ABIAlignment = DAG.getDataLayout().getABITypeAlign(Ty); 15259 if (LD->isUnindexed() && VT.isVector() && 15260 ((Subtarget.hasAltivec() && ISD::isNON_EXTLoad(N) && 15261 // P8 and later hardware should just use LOAD. 15262 !Subtarget.hasP8Vector() && 15263 (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 || 15264 VT == MVT::v4f32))) && 15265 LD->getAlign() < ABIAlignment) { 15266 // This is a type-legal unaligned Altivec load. 15267 SDValue Chain = LD->getChain(); 15268 SDValue Ptr = LD->getBasePtr(); 15269 bool isLittleEndian = Subtarget.isLittleEndian(); 15270 15271 // This implements the loading of unaligned vectors as described in 15272 // the venerable Apple Velocity Engine overview. Specifically: 15273 // https://developer.apple.com/hardwaredrivers/ve/alignment.html 15274 // https://developer.apple.com/hardwaredrivers/ve/code_optimization.html 15275 // 15276 // The general idea is to expand a sequence of one or more unaligned 15277 // loads into an alignment-based permutation-control instruction (lvsl 15278 // or lvsr), a series of regular vector loads (which always truncate 15279 // their input address to an aligned address), and a series of 15280 // permutations. The results of these permutations are the requested 15281 // loaded values. The trick is that the last "extra" load is not taken 15282 // from the address you might suspect (sizeof(vector) bytes after the 15283 // last requested load), but rather sizeof(vector) - 1 bytes after the 15284 // last requested vector. The point of this is to avoid a page fault if 15285 // the base address happened to be aligned. This works because if the 15286 // base address is aligned, then adding less than a full vector length 15287 // will cause the last vector in the sequence to be (re)loaded. 15288 // Otherwise, the next vector will be fetched as you might suspect was 15289 // necessary. 15290 15291 // We might be able to reuse the permutation generation from 15292 // a different base address offset from this one by an aligned amount. 15293 // The INTRINSIC_WO_CHAIN DAG combine will attempt to perform this 15294 // optimization later. 15295 Intrinsic::ID Intr, IntrLD, IntrPerm; 15296 MVT PermCntlTy, PermTy, LDTy; 15297 Intr = isLittleEndian ? Intrinsic::ppc_altivec_lvsr 15298 : Intrinsic::ppc_altivec_lvsl; 15299 IntrLD = Intrinsic::ppc_altivec_lvx; 15300 IntrPerm = Intrinsic::ppc_altivec_vperm; 15301 PermCntlTy = MVT::v16i8; 15302 PermTy = MVT::v4i32; 15303 LDTy = MVT::v4i32; 15304 15305 SDValue PermCntl = BuildIntrinsicOp(Intr, Ptr, DAG, dl, PermCntlTy); 15306 15307 // Create the new MMO for the new base load. It is like the original MMO, 15308 // but represents an area in memory almost twice the vector size centered 15309 // on the original address. If the address is unaligned, we might start 15310 // reading up to (sizeof(vector)-1) bytes below the address of the 15311 // original unaligned load. 15312 MachineFunction &MF = DAG.getMachineFunction(); 15313 MachineMemOperand *BaseMMO = 15314 MF.getMachineMemOperand(LD->getMemOperand(), 15315 -(long)MemVT.getStoreSize()+1, 15316 2*MemVT.getStoreSize()-1); 15317 15318 // Create the new base load. 15319 SDValue LDXIntID = 15320 DAG.getTargetConstant(IntrLD, dl, getPointerTy(MF.getDataLayout())); 15321 SDValue BaseLoadOps[] = { Chain, LDXIntID, Ptr }; 15322 SDValue BaseLoad = 15323 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, dl, 15324 DAG.getVTList(PermTy, MVT::Other), 15325 BaseLoadOps, LDTy, BaseMMO); 15326 15327 // Note that the value of IncOffset (which is provided to the next 15328 // load's pointer info offset value, and thus used to calculate the 15329 // alignment), and the value of IncValue (which is actually used to 15330 // increment the pointer value) are different! This is because we 15331 // require the next load to appear to be aligned, even though it 15332 // is actually offset from the base pointer by a lesser amount. 15333 int IncOffset = VT.getSizeInBits() / 8; 15334 int IncValue = IncOffset; 15335 15336 // Walk (both up and down) the chain looking for another load at the real 15337 // (aligned) offset (the alignment of the other load does not matter in 15338 // this case). If found, then do not use the offset reduction trick, as 15339 // that will prevent the loads from being later combined (as they would 15340 // otherwise be duplicates). 15341 if (!findConsecutiveLoad(LD, DAG)) 15342 --IncValue; 15343 15344 SDValue Increment = 15345 DAG.getConstant(IncValue, dl, getPointerTy(MF.getDataLayout())); 15346 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); 15347 15348 MachineMemOperand *ExtraMMO = 15349 MF.getMachineMemOperand(LD->getMemOperand(), 15350 1, 2*MemVT.getStoreSize()-1); 15351 SDValue ExtraLoadOps[] = { Chain, LDXIntID, Ptr }; 15352 SDValue ExtraLoad = 15353 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, dl, 15354 DAG.getVTList(PermTy, MVT::Other), 15355 ExtraLoadOps, LDTy, ExtraMMO); 15356 15357 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 15358 BaseLoad.getValue(1), ExtraLoad.getValue(1)); 15359 15360 // Because vperm has a big-endian bias, we must reverse the order 15361 // of the input vectors and complement the permute control vector 15362 // when generating little endian code. We have already handled the 15363 // latter by using lvsr instead of lvsl, so just reverse BaseLoad 15364 // and ExtraLoad here. 15365 SDValue Perm; 15366 if (isLittleEndian) 15367 Perm = BuildIntrinsicOp(IntrPerm, 15368 ExtraLoad, BaseLoad, PermCntl, DAG, dl); 15369 else 15370 Perm = BuildIntrinsicOp(IntrPerm, 15371 BaseLoad, ExtraLoad, PermCntl, DAG, dl); 15372 15373 if (VT != PermTy) 15374 Perm = Subtarget.hasAltivec() 15375 ? DAG.getNode(ISD::BITCAST, dl, VT, Perm) 15376 : DAG.getNode(ISD::FP_ROUND, dl, VT, Perm, 15377 DAG.getTargetConstant(1, dl, MVT::i64)); 15378 // second argument is 1 because this rounding 15379 // is always exact. 15380 15381 // The output of the permutation is our loaded result, the TokenFactor is 15382 // our new chain. 15383 DCI.CombineTo(N, Perm, TF); 15384 return SDValue(N, 0); 15385 } 15386 } 15387 break; 15388 case ISD::INTRINSIC_WO_CHAIN: { 15389 bool isLittleEndian = Subtarget.isLittleEndian(); 15390 unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 15391 Intrinsic::ID Intr = (isLittleEndian ? Intrinsic::ppc_altivec_lvsr 15392 : Intrinsic::ppc_altivec_lvsl); 15393 if (IID == Intr && N->getOperand(1)->getOpcode() == ISD::ADD) { 15394 SDValue Add = N->getOperand(1); 15395 15396 int Bits = 4 /* 16 byte alignment */; 15397 15398 if (DAG.MaskedValueIsZero(Add->getOperand(1), 15399 APInt::getAllOnes(Bits /* alignment */) 15400 .zext(Add.getScalarValueSizeInBits()))) { 15401 SDNode *BasePtr = Add->getOperand(0).getNode(); 15402 for (SDNode *U : BasePtr->uses()) { 15403 if (U->getOpcode() == ISD::INTRINSIC_WO_CHAIN && 15404 cast<ConstantSDNode>(U->getOperand(0))->getZExtValue() == IID) { 15405 // We've found another LVSL/LVSR, and this address is an aligned 15406 // multiple of that one. The results will be the same, so use the 15407 // one we've just found instead. 15408 15409 return SDValue(U, 0); 15410 } 15411 } 15412 } 15413 15414 if (isa<ConstantSDNode>(Add->getOperand(1))) { 15415 SDNode *BasePtr = Add->getOperand(0).getNode(); 15416 for (SDNode *U : BasePtr->uses()) { 15417 if (U->getOpcode() == ISD::ADD && 15418 isa<ConstantSDNode>(U->getOperand(1)) && 15419 (cast<ConstantSDNode>(Add->getOperand(1))->getZExtValue() - 15420 cast<ConstantSDNode>(U->getOperand(1))->getZExtValue()) % 15421 (1ULL << Bits) == 15422 0) { 15423 SDNode *OtherAdd = U; 15424 for (SDNode *V : OtherAdd->uses()) { 15425 if (V->getOpcode() == ISD::INTRINSIC_WO_CHAIN && 15426 cast<ConstantSDNode>(V->getOperand(0))->getZExtValue() == 15427 IID) { 15428 return SDValue(V, 0); 15429 } 15430 } 15431 } 15432 } 15433 } 15434 } 15435 15436 // Combine vmaxsw/h/b(a, a's negation) to abs(a) 15437 // Expose the vabsduw/h/b opportunity for down stream 15438 if (!DCI.isAfterLegalizeDAG() && Subtarget.hasP9Altivec() && 15439 (IID == Intrinsic::ppc_altivec_vmaxsw || 15440 IID == Intrinsic::ppc_altivec_vmaxsh || 15441 IID == Intrinsic::ppc_altivec_vmaxsb)) { 15442 SDValue V1 = N->getOperand(1); 15443 SDValue V2 = N->getOperand(2); 15444 if ((V1.getSimpleValueType() == MVT::v4i32 || 15445 V1.getSimpleValueType() == MVT::v8i16 || 15446 V1.getSimpleValueType() == MVT::v16i8) && 15447 V1.getSimpleValueType() == V2.getSimpleValueType()) { 15448 // (0-a, a) 15449 if (V1.getOpcode() == ISD::SUB && 15450 ISD::isBuildVectorAllZeros(V1.getOperand(0).getNode()) && 15451 V1.getOperand(1) == V2) { 15452 return DAG.getNode(ISD::ABS, dl, V2.getValueType(), V2); 15453 } 15454 // (a, 0-a) 15455 if (V2.getOpcode() == ISD::SUB && 15456 ISD::isBuildVectorAllZeros(V2.getOperand(0).getNode()) && 15457 V2.getOperand(1) == V1) { 15458 return DAG.getNode(ISD::ABS, dl, V1.getValueType(), V1); 15459 } 15460 // (x-y, y-x) 15461 if (V1.getOpcode() == ISD::SUB && V2.getOpcode() == ISD::SUB && 15462 V1.getOperand(0) == V2.getOperand(1) && 15463 V1.getOperand(1) == V2.getOperand(0)) { 15464 return DAG.getNode(ISD::ABS, dl, V1.getValueType(), V1); 15465 } 15466 } 15467 } 15468 } 15469 15470 break; 15471 case ISD::INTRINSIC_W_CHAIN: 15472 // For little endian, VSX loads require generating lxvd2x/xxswapd. 15473 // Not needed on ISA 3.0 based CPUs since we have a non-permuting load. 15474 if (Subtarget.needsSwapsForVSXMemOps()) { 15475 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) { 15476 default: 15477 break; 15478 case Intrinsic::ppc_vsx_lxvw4x: 15479 case Intrinsic::ppc_vsx_lxvd2x: 15480 return expandVSXLoadForLE(N, DCI); 15481 } 15482 } 15483 break; 15484 case ISD::INTRINSIC_VOID: 15485 // For little endian, VSX stores require generating xxswapd/stxvd2x. 15486 // Not needed on ISA 3.0 based CPUs since we have a non-permuting store. 15487 if (Subtarget.needsSwapsForVSXMemOps()) { 15488 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) { 15489 default: 15490 break; 15491 case Intrinsic::ppc_vsx_stxvw4x: 15492 case Intrinsic::ppc_vsx_stxvd2x: 15493 return expandVSXStoreForLE(N, DCI); 15494 } 15495 } 15496 break; 15497 case ISD::BSWAP: { 15498 // Turn BSWAP (LOAD) -> lhbrx/lwbrx. 15499 // For subtargets without LDBRX, we can still do better than the default 15500 // expansion even for 64-bit BSWAP (LOAD). 15501 bool Is64BitBswapOn64BitTgt = 15502 Subtarget.isPPC64() && N->getValueType(0) == MVT::i64; 15503 bool IsSingleUseNormalLd = ISD::isNormalLoad(N->getOperand(0).getNode()) && 15504 N->getOperand(0).hasOneUse(); 15505 if (IsSingleUseNormalLd && 15506 (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i16 || 15507 (Subtarget.hasLDBRX() && Is64BitBswapOn64BitTgt))) { 15508 SDValue Load = N->getOperand(0); 15509 LoadSDNode *LD = cast<LoadSDNode>(Load); 15510 // Create the byte-swapping load. 15511 SDValue Ops[] = { 15512 LD->getChain(), // Chain 15513 LD->getBasePtr(), // Ptr 15514 DAG.getValueType(N->getValueType(0)) // VT 15515 }; 15516 SDValue BSLoad = 15517 DAG.getMemIntrinsicNode(PPCISD::LBRX, dl, 15518 DAG.getVTList(N->getValueType(0) == MVT::i64 ? 15519 MVT::i64 : MVT::i32, MVT::Other), 15520 Ops, LD->getMemoryVT(), LD->getMemOperand()); 15521 15522 // If this is an i16 load, insert the truncate. 15523 SDValue ResVal = BSLoad; 15524 if (N->getValueType(0) == MVT::i16) 15525 ResVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, BSLoad); 15526 15527 // First, combine the bswap away. This makes the value produced by the 15528 // load dead. 15529 DCI.CombineTo(N, ResVal); 15530 15531 // Next, combine the load away, we give it a bogus result value but a real 15532 // chain result. The result value is dead because the bswap is dead. 15533 DCI.CombineTo(Load.getNode(), ResVal, BSLoad.getValue(1)); 15534 15535 // Return N so it doesn't get rechecked! 15536 return SDValue(N, 0); 15537 } 15538 // Convert this to two 32-bit bswap loads and a BUILD_PAIR. Do this only 15539 // before legalization so that the BUILD_PAIR is handled correctly. 15540 if (!DCI.isBeforeLegalize() || !Is64BitBswapOn64BitTgt || 15541 !IsSingleUseNormalLd) 15542 return SDValue(); 15543 LoadSDNode *LD = cast<LoadSDNode>(N->getOperand(0)); 15544 15545 // Can't split volatile or atomic loads. 15546 if (!LD->isSimple()) 15547 return SDValue(); 15548 SDValue BasePtr = LD->getBasePtr(); 15549 SDValue Lo = DAG.getLoad(MVT::i32, dl, LD->getChain(), BasePtr, 15550 LD->getPointerInfo(), LD->getAlignment()); 15551 Lo = DAG.getNode(ISD::BSWAP, dl, MVT::i32, Lo); 15552 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 15553 DAG.getIntPtrConstant(4, dl)); 15554 MachineMemOperand *NewMMO = DAG.getMachineFunction().getMachineMemOperand( 15555 LD->getMemOperand(), 4, 4); 15556 SDValue Hi = DAG.getLoad(MVT::i32, dl, LD->getChain(), BasePtr, NewMMO); 15557 Hi = DAG.getNode(ISD::BSWAP, dl, MVT::i32, Hi); 15558 SDValue Res; 15559 if (Subtarget.isLittleEndian()) 15560 Res = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Hi, Lo); 15561 else 15562 Res = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi); 15563 SDValue TF = 15564 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 15565 Hi.getOperand(0).getValue(1), Lo.getOperand(0).getValue(1)); 15566 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), TF); 15567 return Res; 15568 } 15569 case PPCISD::VCMP: 15570 // If a VCMP_rec node already exists with exactly the same operands as this 15571 // node, use its result instead of this node (VCMP_rec computes both a CR6 15572 // and a normal output). 15573 // 15574 if (!N->getOperand(0).hasOneUse() && 15575 !N->getOperand(1).hasOneUse() && 15576 !N->getOperand(2).hasOneUse()) { 15577 15578 // Scan all of the users of the LHS, looking for VCMP_rec's that match. 15579 SDNode *VCMPrecNode = nullptr; 15580 15581 SDNode *LHSN = N->getOperand(0).getNode(); 15582 for (SDNode::use_iterator UI = LHSN->use_begin(), E = LHSN->use_end(); 15583 UI != E; ++UI) 15584 if (UI->getOpcode() == PPCISD::VCMP_rec && 15585 UI->getOperand(1) == N->getOperand(1) && 15586 UI->getOperand(2) == N->getOperand(2) && 15587 UI->getOperand(0) == N->getOperand(0)) { 15588 VCMPrecNode = *UI; 15589 break; 15590 } 15591 15592 // If there is no VCMP_rec node, or if the flag value has a single use, 15593 // don't transform this. 15594 if (!VCMPrecNode || VCMPrecNode->hasNUsesOfValue(0, 1)) 15595 break; 15596 15597 // Look at the (necessarily single) use of the flag value. If it has a 15598 // chain, this transformation is more complex. Note that multiple things 15599 // could use the value result, which we should ignore. 15600 SDNode *FlagUser = nullptr; 15601 for (SDNode::use_iterator UI = VCMPrecNode->use_begin(); 15602 FlagUser == nullptr; ++UI) { 15603 assert(UI != VCMPrecNode->use_end() && "Didn't find user!"); 15604 SDNode *User = *UI; 15605 for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) { 15606 if (User->getOperand(i) == SDValue(VCMPrecNode, 1)) { 15607 FlagUser = User; 15608 break; 15609 } 15610 } 15611 } 15612 15613 // If the user is a MFOCRF instruction, we know this is safe. 15614 // Otherwise we give up for right now. 15615 if (FlagUser->getOpcode() == PPCISD::MFOCRF) 15616 return SDValue(VCMPrecNode, 0); 15617 } 15618 break; 15619 case ISD::BRCOND: { 15620 SDValue Cond = N->getOperand(1); 15621 SDValue Target = N->getOperand(2); 15622 15623 if (Cond.getOpcode() == ISD::INTRINSIC_W_CHAIN && 15624 cast<ConstantSDNode>(Cond.getOperand(1))->getZExtValue() == 15625 Intrinsic::loop_decrement) { 15626 15627 // We now need to make the intrinsic dead (it cannot be instruction 15628 // selected). 15629 DAG.ReplaceAllUsesOfValueWith(Cond.getValue(1), Cond.getOperand(0)); 15630 assert(Cond.getNode()->hasOneUse() && 15631 "Counter decrement has more than one use"); 15632 15633 return DAG.getNode(PPCISD::BDNZ, dl, MVT::Other, 15634 N->getOperand(0), Target); 15635 } 15636 } 15637 break; 15638 case ISD::BR_CC: { 15639 // If this is a branch on an altivec predicate comparison, lower this so 15640 // that we don't have to do a MFOCRF: instead, branch directly on CR6. This 15641 // lowering is done pre-legalize, because the legalizer lowers the predicate 15642 // compare down to code that is difficult to reassemble. 15643 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get(); 15644 SDValue LHS = N->getOperand(2), RHS = N->getOperand(3); 15645 15646 // Sometimes the promoted value of the intrinsic is ANDed by some non-zero 15647 // value. If so, pass-through the AND to get to the intrinsic. 15648 if (LHS.getOpcode() == ISD::AND && 15649 LHS.getOperand(0).getOpcode() == ISD::INTRINSIC_W_CHAIN && 15650 cast<ConstantSDNode>(LHS.getOperand(0).getOperand(1))->getZExtValue() == 15651 Intrinsic::loop_decrement && 15652 isa<ConstantSDNode>(LHS.getOperand(1)) && 15653 !isNullConstant(LHS.getOperand(1))) 15654 LHS = LHS.getOperand(0); 15655 15656 if (LHS.getOpcode() == ISD::INTRINSIC_W_CHAIN && 15657 cast<ConstantSDNode>(LHS.getOperand(1))->getZExtValue() == 15658 Intrinsic::loop_decrement && 15659 isa<ConstantSDNode>(RHS)) { 15660 assert((CC == ISD::SETEQ || CC == ISD::SETNE) && 15661 "Counter decrement comparison is not EQ or NE"); 15662 15663 unsigned Val = cast<ConstantSDNode>(RHS)->getZExtValue(); 15664 bool isBDNZ = (CC == ISD::SETEQ && Val) || 15665 (CC == ISD::SETNE && !Val); 15666 15667 // We now need to make the intrinsic dead (it cannot be instruction 15668 // selected). 15669 DAG.ReplaceAllUsesOfValueWith(LHS.getValue(1), LHS.getOperand(0)); 15670 assert(LHS.getNode()->hasOneUse() && 15671 "Counter decrement has more than one use"); 15672 15673 return DAG.getNode(isBDNZ ? PPCISD::BDNZ : PPCISD::BDZ, dl, MVT::Other, 15674 N->getOperand(0), N->getOperand(4)); 15675 } 15676 15677 int CompareOpc; 15678 bool isDot; 15679 15680 if (LHS.getOpcode() == ISD::INTRINSIC_WO_CHAIN && 15681 isa<ConstantSDNode>(RHS) && (CC == ISD::SETEQ || CC == ISD::SETNE) && 15682 getVectorCompareInfo(LHS, CompareOpc, isDot, Subtarget)) { 15683 assert(isDot && "Can't compare against a vector result!"); 15684 15685 // If this is a comparison against something other than 0/1, then we know 15686 // that the condition is never/always true. 15687 unsigned Val = cast<ConstantSDNode>(RHS)->getZExtValue(); 15688 if (Val != 0 && Val != 1) { 15689 if (CC == ISD::SETEQ) // Cond never true, remove branch. 15690 return N->getOperand(0); 15691 // Always !=, turn it into an unconditional branch. 15692 return DAG.getNode(ISD::BR, dl, MVT::Other, 15693 N->getOperand(0), N->getOperand(4)); 15694 } 15695 15696 bool BranchOnWhenPredTrue = (CC == ISD::SETEQ) ^ (Val == 0); 15697 15698 // Create the PPCISD altivec 'dot' comparison node. 15699 SDValue Ops[] = { 15700 LHS.getOperand(2), // LHS of compare 15701 LHS.getOperand(3), // RHS of compare 15702 DAG.getConstant(CompareOpc, dl, MVT::i32) 15703 }; 15704 EVT VTs[] = { LHS.getOperand(2).getValueType(), MVT::Glue }; 15705 SDValue CompNode = DAG.getNode(PPCISD::VCMP_rec, dl, VTs, Ops); 15706 15707 // Unpack the result based on how the target uses it. 15708 PPC::Predicate CompOpc; 15709 switch (cast<ConstantSDNode>(LHS.getOperand(1))->getZExtValue()) { 15710 default: // Can't happen, don't crash on invalid number though. 15711 case 0: // Branch on the value of the EQ bit of CR6. 15712 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_EQ : PPC::PRED_NE; 15713 break; 15714 case 1: // Branch on the inverted value of the EQ bit of CR6. 15715 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_NE : PPC::PRED_EQ; 15716 break; 15717 case 2: // Branch on the value of the LT bit of CR6. 15718 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_LT : PPC::PRED_GE; 15719 break; 15720 case 3: // Branch on the inverted value of the LT bit of CR6. 15721 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_GE : PPC::PRED_LT; 15722 break; 15723 } 15724 15725 return DAG.getNode(PPCISD::COND_BRANCH, dl, MVT::Other, N->getOperand(0), 15726 DAG.getConstant(CompOpc, dl, MVT::i32), 15727 DAG.getRegister(PPC::CR6, MVT::i32), 15728 N->getOperand(4), CompNode.getValue(1)); 15729 } 15730 break; 15731 } 15732 case ISD::BUILD_VECTOR: 15733 return DAGCombineBuildVector(N, DCI); 15734 case ISD::ABS: 15735 return combineABS(N, DCI); 15736 case ISD::VSELECT: 15737 return combineVSelect(N, DCI); 15738 } 15739 15740 return SDValue(); 15741 } 15742 15743 SDValue 15744 PPCTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor, 15745 SelectionDAG &DAG, 15746 SmallVectorImpl<SDNode *> &Created) const { 15747 // fold (sdiv X, pow2) 15748 EVT VT = N->getValueType(0); 15749 if (VT == MVT::i64 && !Subtarget.isPPC64()) 15750 return SDValue(); 15751 if ((VT != MVT::i32 && VT != MVT::i64) || 15752 !(Divisor.isPowerOf2() || Divisor.isNegatedPowerOf2())) 15753 return SDValue(); 15754 15755 SDLoc DL(N); 15756 SDValue N0 = N->getOperand(0); 15757 15758 bool IsNegPow2 = Divisor.isNegatedPowerOf2(); 15759 unsigned Lg2 = (IsNegPow2 ? -Divisor : Divisor).countTrailingZeros(); 15760 SDValue ShiftAmt = DAG.getConstant(Lg2, DL, VT); 15761 15762 SDValue Op = DAG.getNode(PPCISD::SRA_ADDZE, DL, VT, N0, ShiftAmt); 15763 Created.push_back(Op.getNode()); 15764 15765 if (IsNegPow2) { 15766 Op = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op); 15767 Created.push_back(Op.getNode()); 15768 } 15769 15770 return Op; 15771 } 15772 15773 //===----------------------------------------------------------------------===// 15774 // Inline Assembly Support 15775 //===----------------------------------------------------------------------===// 15776 15777 void PPCTargetLowering::computeKnownBitsForTargetNode(const SDValue Op, 15778 KnownBits &Known, 15779 const APInt &DemandedElts, 15780 const SelectionDAG &DAG, 15781 unsigned Depth) const { 15782 Known.resetAll(); 15783 switch (Op.getOpcode()) { 15784 default: break; 15785 case PPCISD::LBRX: { 15786 // lhbrx is known to have the top bits cleared out. 15787 if (cast<VTSDNode>(Op.getOperand(2))->getVT() == MVT::i16) 15788 Known.Zero = 0xFFFF0000; 15789 break; 15790 } 15791 case ISD::INTRINSIC_WO_CHAIN: { 15792 switch (cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue()) { 15793 default: break; 15794 case Intrinsic::ppc_altivec_vcmpbfp_p: 15795 case Intrinsic::ppc_altivec_vcmpeqfp_p: 15796 case Intrinsic::ppc_altivec_vcmpequb_p: 15797 case Intrinsic::ppc_altivec_vcmpequh_p: 15798 case Intrinsic::ppc_altivec_vcmpequw_p: 15799 case Intrinsic::ppc_altivec_vcmpequd_p: 15800 case Intrinsic::ppc_altivec_vcmpequq_p: 15801 case Intrinsic::ppc_altivec_vcmpgefp_p: 15802 case Intrinsic::ppc_altivec_vcmpgtfp_p: 15803 case Intrinsic::ppc_altivec_vcmpgtsb_p: 15804 case Intrinsic::ppc_altivec_vcmpgtsh_p: 15805 case Intrinsic::ppc_altivec_vcmpgtsw_p: 15806 case Intrinsic::ppc_altivec_vcmpgtsd_p: 15807 case Intrinsic::ppc_altivec_vcmpgtsq_p: 15808 case Intrinsic::ppc_altivec_vcmpgtub_p: 15809 case Intrinsic::ppc_altivec_vcmpgtuh_p: 15810 case Intrinsic::ppc_altivec_vcmpgtuw_p: 15811 case Intrinsic::ppc_altivec_vcmpgtud_p: 15812 case Intrinsic::ppc_altivec_vcmpgtuq_p: 15813 Known.Zero = ~1U; // All bits but the low one are known to be zero. 15814 break; 15815 } 15816 break; 15817 } 15818 case ISD::INTRINSIC_W_CHAIN: { 15819 switch (cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue()) { 15820 default: 15821 break; 15822 case Intrinsic::ppc_load2r: 15823 // Top bits are cleared for load2r (which is the same as lhbrx). 15824 Known.Zero = 0xFFFF0000; 15825 break; 15826 } 15827 break; 15828 } 15829 } 15830 } 15831 15832 Align PPCTargetLowering::getPrefLoopAlignment(MachineLoop *ML) const { 15833 switch (Subtarget.getCPUDirective()) { 15834 default: break; 15835 case PPC::DIR_970: 15836 case PPC::DIR_PWR4: 15837 case PPC::DIR_PWR5: 15838 case PPC::DIR_PWR5X: 15839 case PPC::DIR_PWR6: 15840 case PPC::DIR_PWR6X: 15841 case PPC::DIR_PWR7: 15842 case PPC::DIR_PWR8: 15843 case PPC::DIR_PWR9: 15844 case PPC::DIR_PWR10: 15845 case PPC::DIR_PWR_FUTURE: { 15846 if (!ML) 15847 break; 15848 15849 if (!DisableInnermostLoopAlign32) { 15850 // If the nested loop is an innermost loop, prefer to a 32-byte alignment, 15851 // so that we can decrease cache misses and branch-prediction misses. 15852 // Actual alignment of the loop will depend on the hotness check and other 15853 // logic in alignBlocks. 15854 if (ML->getLoopDepth() > 1 && ML->getSubLoops().empty()) 15855 return Align(32); 15856 } 15857 15858 const PPCInstrInfo *TII = Subtarget.getInstrInfo(); 15859 15860 // For small loops (between 5 and 8 instructions), align to a 32-byte 15861 // boundary so that the entire loop fits in one instruction-cache line. 15862 uint64_t LoopSize = 0; 15863 for (auto I = ML->block_begin(), IE = ML->block_end(); I != IE; ++I) 15864 for (auto J = (*I)->begin(), JE = (*I)->end(); J != JE; ++J) { 15865 LoopSize += TII->getInstSizeInBytes(*J); 15866 if (LoopSize > 32) 15867 break; 15868 } 15869 15870 if (LoopSize > 16 && LoopSize <= 32) 15871 return Align(32); 15872 15873 break; 15874 } 15875 } 15876 15877 return TargetLowering::getPrefLoopAlignment(ML); 15878 } 15879 15880 /// getConstraintType - Given a constraint, return the type of 15881 /// constraint it is for this target. 15882 PPCTargetLowering::ConstraintType 15883 PPCTargetLowering::getConstraintType(StringRef Constraint) const { 15884 if (Constraint.size() == 1) { 15885 switch (Constraint[0]) { 15886 default: break; 15887 case 'b': 15888 case 'r': 15889 case 'f': 15890 case 'd': 15891 case 'v': 15892 case 'y': 15893 return C_RegisterClass; 15894 case 'Z': 15895 // FIXME: While Z does indicate a memory constraint, it specifically 15896 // indicates an r+r address (used in conjunction with the 'y' modifier 15897 // in the replacement string). Currently, we're forcing the base 15898 // register to be r0 in the asm printer (which is interpreted as zero) 15899 // and forming the complete address in the second register. This is 15900 // suboptimal. 15901 return C_Memory; 15902 } 15903 } else if (Constraint == "wc") { // individual CR bits. 15904 return C_RegisterClass; 15905 } else if (Constraint == "wa" || Constraint == "wd" || 15906 Constraint == "wf" || Constraint == "ws" || 15907 Constraint == "wi" || Constraint == "ww") { 15908 return C_RegisterClass; // VSX registers. 15909 } 15910 return TargetLowering::getConstraintType(Constraint); 15911 } 15912 15913 /// Examine constraint type and operand type and determine a weight value. 15914 /// This object must already have been set up with the operand type 15915 /// and the current alternative constraint selected. 15916 TargetLowering::ConstraintWeight 15917 PPCTargetLowering::getSingleConstraintMatchWeight( 15918 AsmOperandInfo &info, const char *constraint) const { 15919 ConstraintWeight weight = CW_Invalid; 15920 Value *CallOperandVal = info.CallOperandVal; 15921 // If we don't have a value, we can't do a match, 15922 // but allow it at the lowest weight. 15923 if (!CallOperandVal) 15924 return CW_Default; 15925 Type *type = CallOperandVal->getType(); 15926 15927 // Look at the constraint type. 15928 if (StringRef(constraint) == "wc" && type->isIntegerTy(1)) 15929 return CW_Register; // an individual CR bit. 15930 else if ((StringRef(constraint) == "wa" || 15931 StringRef(constraint) == "wd" || 15932 StringRef(constraint) == "wf") && 15933 type->isVectorTy()) 15934 return CW_Register; 15935 else if (StringRef(constraint) == "wi" && type->isIntegerTy(64)) 15936 return CW_Register; // just hold 64-bit integers data. 15937 else if (StringRef(constraint) == "ws" && type->isDoubleTy()) 15938 return CW_Register; 15939 else if (StringRef(constraint) == "ww" && type->isFloatTy()) 15940 return CW_Register; 15941 15942 switch (*constraint) { 15943 default: 15944 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); 15945 break; 15946 case 'b': 15947 if (type->isIntegerTy()) 15948 weight = CW_Register; 15949 break; 15950 case 'f': 15951 if (type->isFloatTy()) 15952 weight = CW_Register; 15953 break; 15954 case 'd': 15955 if (type->isDoubleTy()) 15956 weight = CW_Register; 15957 break; 15958 case 'v': 15959 if (type->isVectorTy()) 15960 weight = CW_Register; 15961 break; 15962 case 'y': 15963 weight = CW_Register; 15964 break; 15965 case 'Z': 15966 weight = CW_Memory; 15967 break; 15968 } 15969 return weight; 15970 } 15971 15972 std::pair<unsigned, const TargetRegisterClass *> 15973 PPCTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, 15974 StringRef Constraint, 15975 MVT VT) const { 15976 if (Constraint.size() == 1) { 15977 // GCC RS6000 Constraint Letters 15978 switch (Constraint[0]) { 15979 case 'b': // R1-R31 15980 if (VT == MVT::i64 && Subtarget.isPPC64()) 15981 return std::make_pair(0U, &PPC::G8RC_NOX0RegClass); 15982 return std::make_pair(0U, &PPC::GPRC_NOR0RegClass); 15983 case 'r': // R0-R31 15984 if (VT == MVT::i64 && Subtarget.isPPC64()) 15985 return std::make_pair(0U, &PPC::G8RCRegClass); 15986 return std::make_pair(0U, &PPC::GPRCRegClass); 15987 // 'd' and 'f' constraints are both defined to be "the floating point 15988 // registers", where one is for 32-bit and the other for 64-bit. We don't 15989 // really care overly much here so just give them all the same reg classes. 15990 case 'd': 15991 case 'f': 15992 if (Subtarget.hasSPE()) { 15993 if (VT == MVT::f32 || VT == MVT::i32) 15994 return std::make_pair(0U, &PPC::GPRCRegClass); 15995 if (VT == MVT::f64 || VT == MVT::i64) 15996 return std::make_pair(0U, &PPC::SPERCRegClass); 15997 } else { 15998 if (VT == MVT::f32 || VT == MVT::i32) 15999 return std::make_pair(0U, &PPC::F4RCRegClass); 16000 if (VT == MVT::f64 || VT == MVT::i64) 16001 return std::make_pair(0U, &PPC::F8RCRegClass); 16002 } 16003 break; 16004 case 'v': 16005 if (Subtarget.hasAltivec() && VT.isVector()) 16006 return std::make_pair(0U, &PPC::VRRCRegClass); 16007 else if (Subtarget.hasVSX()) 16008 // Scalars in Altivec registers only make sense with VSX. 16009 return std::make_pair(0U, &PPC::VFRCRegClass); 16010 break; 16011 case 'y': // crrc 16012 return std::make_pair(0U, &PPC::CRRCRegClass); 16013 } 16014 } else if (Constraint == "wc" && Subtarget.useCRBits()) { 16015 // An individual CR bit. 16016 return std::make_pair(0U, &PPC::CRBITRCRegClass); 16017 } else if ((Constraint == "wa" || Constraint == "wd" || 16018 Constraint == "wf" || Constraint == "wi") && 16019 Subtarget.hasVSX()) { 16020 // A VSX register for either a scalar (FP) or vector. There is no 16021 // support for single precision scalars on subtargets prior to Power8. 16022 if (VT.isVector()) 16023 return std::make_pair(0U, &PPC::VSRCRegClass); 16024 if (VT == MVT::f32 && Subtarget.hasP8Vector()) 16025 return std::make_pair(0U, &PPC::VSSRCRegClass); 16026 return std::make_pair(0U, &PPC::VSFRCRegClass); 16027 } else if ((Constraint == "ws" || Constraint == "ww") && Subtarget.hasVSX()) { 16028 if (VT == MVT::f32 && Subtarget.hasP8Vector()) 16029 return std::make_pair(0U, &PPC::VSSRCRegClass); 16030 else 16031 return std::make_pair(0U, &PPC::VSFRCRegClass); 16032 } else if (Constraint == "lr") { 16033 if (VT == MVT::i64) 16034 return std::make_pair(0U, &PPC::LR8RCRegClass); 16035 else 16036 return std::make_pair(0U, &PPC::LRRCRegClass); 16037 } 16038 16039 // Handle special cases of physical registers that are not properly handled 16040 // by the base class. 16041 if (Constraint[0] == '{' && Constraint[Constraint.size() - 1] == '}') { 16042 // If we name a VSX register, we can't defer to the base class because it 16043 // will not recognize the correct register (their names will be VSL{0-31} 16044 // and V{0-31} so they won't match). So we match them here. 16045 if (Constraint.size() > 3 && Constraint[1] == 'v' && Constraint[2] == 's') { 16046 int VSNum = atoi(Constraint.data() + 3); 16047 assert(VSNum >= 0 && VSNum <= 63 && 16048 "Attempted to access a vsr out of range"); 16049 if (VSNum < 32) 16050 return std::make_pair(PPC::VSL0 + VSNum, &PPC::VSRCRegClass); 16051 return std::make_pair(PPC::V0 + VSNum - 32, &PPC::VSRCRegClass); 16052 } 16053 16054 // For float registers, we can't defer to the base class as it will match 16055 // the SPILLTOVSRRC class. 16056 if (Constraint.size() > 3 && Constraint[1] == 'f') { 16057 int RegNum = atoi(Constraint.data() + 2); 16058 if (RegNum > 31 || RegNum < 0) 16059 report_fatal_error("Invalid floating point register number"); 16060 if (VT == MVT::f32 || VT == MVT::i32) 16061 return Subtarget.hasSPE() 16062 ? std::make_pair(PPC::R0 + RegNum, &PPC::GPRCRegClass) 16063 : std::make_pair(PPC::F0 + RegNum, &PPC::F4RCRegClass); 16064 if (VT == MVT::f64 || VT == MVT::i64) 16065 return Subtarget.hasSPE() 16066 ? std::make_pair(PPC::S0 + RegNum, &PPC::SPERCRegClass) 16067 : std::make_pair(PPC::F0 + RegNum, &PPC::F8RCRegClass); 16068 } 16069 } 16070 16071 std::pair<unsigned, const TargetRegisterClass *> R = 16072 TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); 16073 16074 // r[0-9]+ are used, on PPC64, to refer to the corresponding 64-bit registers 16075 // (which we call X[0-9]+). If a 64-bit value has been requested, and a 16076 // 32-bit GPR has been selected, then 'upgrade' it to the 64-bit parent 16077 // register. 16078 // FIXME: If TargetLowering::getRegForInlineAsmConstraint could somehow use 16079 // the AsmName field from *RegisterInfo.td, then this would not be necessary. 16080 if (R.first && VT == MVT::i64 && Subtarget.isPPC64() && 16081 PPC::GPRCRegClass.contains(R.first)) 16082 return std::make_pair(TRI->getMatchingSuperReg(R.first, 16083 PPC::sub_32, &PPC::G8RCRegClass), 16084 &PPC::G8RCRegClass); 16085 16086 // GCC accepts 'cc' as an alias for 'cr0', and we need to do the same. 16087 if (!R.second && StringRef("{cc}").equals_insensitive(Constraint)) { 16088 R.first = PPC::CR0; 16089 R.second = &PPC::CRRCRegClass; 16090 } 16091 // FIXME: This warning should ideally be emitted in the front end. 16092 const auto &TM = getTargetMachine(); 16093 if (Subtarget.isAIXABI() && !TM.getAIXExtendedAltivecABI()) { 16094 if (((R.first >= PPC::V20 && R.first <= PPC::V31) || 16095 (R.first >= PPC::VF20 && R.first <= PPC::VF31)) && 16096 (R.second == &PPC::VSRCRegClass || R.second == &PPC::VSFRCRegClass)) 16097 errs() << "warning: vector registers 20 to 32 are reserved in the " 16098 "default AIX AltiVec ABI and cannot be used\n"; 16099 } 16100 16101 return R; 16102 } 16103 16104 /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops 16105 /// vector. If it is invalid, don't add anything to Ops. 16106 void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op, 16107 std::string &Constraint, 16108 std::vector<SDValue>&Ops, 16109 SelectionDAG &DAG) const { 16110 SDValue Result; 16111 16112 // Only support length 1 constraints. 16113 if (Constraint.length() > 1) return; 16114 16115 char Letter = Constraint[0]; 16116 switch (Letter) { 16117 default: break; 16118 case 'I': 16119 case 'J': 16120 case 'K': 16121 case 'L': 16122 case 'M': 16123 case 'N': 16124 case 'O': 16125 case 'P': { 16126 ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op); 16127 if (!CST) return; // Must be an immediate to match. 16128 SDLoc dl(Op); 16129 int64_t Value = CST->getSExtValue(); 16130 EVT TCVT = MVT::i64; // All constants taken to be 64 bits so that negative 16131 // numbers are printed as such. 16132 switch (Letter) { 16133 default: llvm_unreachable("Unknown constraint letter!"); 16134 case 'I': // "I" is a signed 16-bit constant. 16135 if (isInt<16>(Value)) 16136 Result = DAG.getTargetConstant(Value, dl, TCVT); 16137 break; 16138 case 'J': // "J" is a constant with only the high-order 16 bits nonzero. 16139 if (isShiftedUInt<16, 16>(Value)) 16140 Result = DAG.getTargetConstant(Value, dl, TCVT); 16141 break; 16142 case 'L': // "L" is a signed 16-bit constant shifted left 16 bits. 16143 if (isShiftedInt<16, 16>(Value)) 16144 Result = DAG.getTargetConstant(Value, dl, TCVT); 16145 break; 16146 case 'K': // "K" is a constant with only the low-order 16 bits nonzero. 16147 if (isUInt<16>(Value)) 16148 Result = DAG.getTargetConstant(Value, dl, TCVT); 16149 break; 16150 case 'M': // "M" is a constant that is greater than 31. 16151 if (Value > 31) 16152 Result = DAG.getTargetConstant(Value, dl, TCVT); 16153 break; 16154 case 'N': // "N" is a positive constant that is an exact power of two. 16155 if (Value > 0 && isPowerOf2_64(Value)) 16156 Result = DAG.getTargetConstant(Value, dl, TCVT); 16157 break; 16158 case 'O': // "O" is the constant zero. 16159 if (Value == 0) 16160 Result = DAG.getTargetConstant(Value, dl, TCVT); 16161 break; 16162 case 'P': // "P" is a constant whose negation is a signed 16-bit constant. 16163 if (isInt<16>(-Value)) 16164 Result = DAG.getTargetConstant(Value, dl, TCVT); 16165 break; 16166 } 16167 break; 16168 } 16169 } 16170 16171 if (Result.getNode()) { 16172 Ops.push_back(Result); 16173 return; 16174 } 16175 16176 // Handle standard constraint letters. 16177 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 16178 } 16179 16180 // isLegalAddressingMode - Return true if the addressing mode represented 16181 // by AM is legal for this target, for a load/store of the specified type. 16182 bool PPCTargetLowering::isLegalAddressingMode(const DataLayout &DL, 16183 const AddrMode &AM, Type *Ty, 16184 unsigned AS, 16185 Instruction *I) const { 16186 // Vector type r+i form is supported since power9 as DQ form. We don't check 16187 // the offset matching DQ form requirement(off % 16 == 0), because on PowerPC, 16188 // imm form is preferred and the offset can be adjusted to use imm form later 16189 // in pass PPCLoopInstrFormPrep. Also in LSR, for one LSRUse, it uses min and 16190 // max offset to check legal addressing mode, we should be a little aggressive 16191 // to contain other offsets for that LSRUse. 16192 if (Ty->isVectorTy() && AM.BaseOffs != 0 && !Subtarget.hasP9Vector()) 16193 return false; 16194 16195 // PPC allows a sign-extended 16-bit immediate field. 16196 if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1) 16197 return false; 16198 16199 // No global is ever allowed as a base. 16200 if (AM.BaseGV) 16201 return false; 16202 16203 // PPC only support r+r, 16204 switch (AM.Scale) { 16205 case 0: // "r+i" or just "i", depending on HasBaseReg. 16206 break; 16207 case 1: 16208 if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed. 16209 return false; 16210 // Otherwise we have r+r or r+i. 16211 break; 16212 case 2: 16213 if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed. 16214 return false; 16215 // Allow 2*r as r+r. 16216 break; 16217 default: 16218 // No other scales are supported. 16219 return false; 16220 } 16221 16222 return true; 16223 } 16224 16225 SDValue PPCTargetLowering::LowerRETURNADDR(SDValue Op, 16226 SelectionDAG &DAG) const { 16227 MachineFunction &MF = DAG.getMachineFunction(); 16228 MachineFrameInfo &MFI = MF.getFrameInfo(); 16229 MFI.setReturnAddressIsTaken(true); 16230 16231 if (verifyReturnAddressArgumentIsConstant(Op, DAG)) 16232 return SDValue(); 16233 16234 SDLoc dl(Op); 16235 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 16236 16237 // Make sure the function does not optimize away the store of the RA to 16238 // the stack. 16239 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 16240 FuncInfo->setLRStoreRequired(); 16241 bool isPPC64 = Subtarget.isPPC64(); 16242 auto PtrVT = getPointerTy(MF.getDataLayout()); 16243 16244 if (Depth > 0) { 16245 // The link register (return address) is saved in the caller's frame 16246 // not the callee's stack frame. So we must get the caller's frame 16247 // address and load the return address at the LR offset from there. 16248 SDValue FrameAddr = 16249 DAG.getLoad(Op.getValueType(), dl, DAG.getEntryNode(), 16250 LowerFRAMEADDR(Op, DAG), MachinePointerInfo()); 16251 SDValue Offset = 16252 DAG.getConstant(Subtarget.getFrameLowering()->getReturnSaveOffset(), dl, 16253 isPPC64 ? MVT::i64 : MVT::i32); 16254 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), 16255 DAG.getNode(ISD::ADD, dl, PtrVT, FrameAddr, Offset), 16256 MachinePointerInfo()); 16257 } 16258 16259 // Just load the return address off the stack. 16260 SDValue RetAddrFI = getReturnAddrFrameIndex(DAG); 16261 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), RetAddrFI, 16262 MachinePointerInfo()); 16263 } 16264 16265 SDValue PPCTargetLowering::LowerFRAMEADDR(SDValue Op, 16266 SelectionDAG &DAG) const { 16267 SDLoc dl(Op); 16268 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 16269 16270 MachineFunction &MF = DAG.getMachineFunction(); 16271 MachineFrameInfo &MFI = MF.getFrameInfo(); 16272 MFI.setFrameAddressIsTaken(true); 16273 16274 EVT PtrVT = getPointerTy(MF.getDataLayout()); 16275 bool isPPC64 = PtrVT == MVT::i64; 16276 16277 // Naked functions never have a frame pointer, and so we use r1. For all 16278 // other functions, this decision must be delayed until during PEI. 16279 unsigned FrameReg; 16280 if (MF.getFunction().hasFnAttribute(Attribute::Naked)) 16281 FrameReg = isPPC64 ? PPC::X1 : PPC::R1; 16282 else 16283 FrameReg = isPPC64 ? PPC::FP8 : PPC::FP; 16284 16285 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, 16286 PtrVT); 16287 while (Depth--) 16288 FrameAddr = DAG.getLoad(Op.getValueType(), dl, DAG.getEntryNode(), 16289 FrameAddr, MachinePointerInfo()); 16290 return FrameAddr; 16291 } 16292 16293 // FIXME? Maybe this could be a TableGen attribute on some registers and 16294 // this table could be generated automatically from RegInfo. 16295 Register PPCTargetLowering::getRegisterByName(const char* RegName, LLT VT, 16296 const MachineFunction &MF) const { 16297 bool isPPC64 = Subtarget.isPPC64(); 16298 16299 bool is64Bit = isPPC64 && VT == LLT::scalar(64); 16300 if (!is64Bit && VT != LLT::scalar(32)) 16301 report_fatal_error("Invalid register global variable type"); 16302 16303 Register Reg = StringSwitch<Register>(RegName) 16304 .Case("r1", is64Bit ? PPC::X1 : PPC::R1) 16305 .Case("r2", isPPC64 ? Register() : PPC::R2) 16306 .Case("r13", (is64Bit ? PPC::X13 : PPC::R13)) 16307 .Default(Register()); 16308 16309 if (Reg) 16310 return Reg; 16311 report_fatal_error("Invalid register name global variable"); 16312 } 16313 16314 bool PPCTargetLowering::isAccessedAsGotIndirect(SDValue GA) const { 16315 // 32-bit SVR4 ABI access everything as got-indirect. 16316 if (Subtarget.is32BitELFABI()) 16317 return true; 16318 16319 // AIX accesses everything indirectly through the TOC, which is similar to 16320 // the GOT. 16321 if (Subtarget.isAIXABI()) 16322 return true; 16323 16324 CodeModel::Model CModel = getTargetMachine().getCodeModel(); 16325 // If it is small or large code model, module locals are accessed 16326 // indirectly by loading their address from .toc/.got. 16327 if (CModel == CodeModel::Small || CModel == CodeModel::Large) 16328 return true; 16329 16330 // JumpTable and BlockAddress are accessed as got-indirect. 16331 if (isa<JumpTableSDNode>(GA) || isa<BlockAddressSDNode>(GA)) 16332 return true; 16333 16334 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(GA)) 16335 return Subtarget.isGVIndirectSymbol(G->getGlobal()); 16336 16337 return false; 16338 } 16339 16340 bool 16341 PPCTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const { 16342 // The PowerPC target isn't yet aware of offsets. 16343 return false; 16344 } 16345 16346 bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, 16347 const CallInst &I, 16348 MachineFunction &MF, 16349 unsigned Intrinsic) const { 16350 switch (Intrinsic) { 16351 case Intrinsic::ppc_atomicrmw_xchg_i128: 16352 case Intrinsic::ppc_atomicrmw_add_i128: 16353 case Intrinsic::ppc_atomicrmw_sub_i128: 16354 case Intrinsic::ppc_atomicrmw_nand_i128: 16355 case Intrinsic::ppc_atomicrmw_and_i128: 16356 case Intrinsic::ppc_atomicrmw_or_i128: 16357 case Intrinsic::ppc_atomicrmw_xor_i128: 16358 case Intrinsic::ppc_cmpxchg_i128: 16359 Info.opc = ISD::INTRINSIC_W_CHAIN; 16360 Info.memVT = MVT::i128; 16361 Info.ptrVal = I.getArgOperand(0); 16362 Info.offset = 0; 16363 Info.align = Align(16); 16364 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore | 16365 MachineMemOperand::MOVolatile; 16366 return true; 16367 case Intrinsic::ppc_atomic_load_i128: 16368 Info.opc = ISD::INTRINSIC_W_CHAIN; 16369 Info.memVT = MVT::i128; 16370 Info.ptrVal = I.getArgOperand(0); 16371 Info.offset = 0; 16372 Info.align = Align(16); 16373 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile; 16374 return true; 16375 case Intrinsic::ppc_atomic_store_i128: 16376 Info.opc = ISD::INTRINSIC_VOID; 16377 Info.memVT = MVT::i128; 16378 Info.ptrVal = I.getArgOperand(2); 16379 Info.offset = 0; 16380 Info.align = Align(16); 16381 Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile; 16382 return true; 16383 case Intrinsic::ppc_altivec_lvx: 16384 case Intrinsic::ppc_altivec_lvxl: 16385 case Intrinsic::ppc_altivec_lvebx: 16386 case Intrinsic::ppc_altivec_lvehx: 16387 case Intrinsic::ppc_altivec_lvewx: 16388 case Intrinsic::ppc_vsx_lxvd2x: 16389 case Intrinsic::ppc_vsx_lxvw4x: 16390 case Intrinsic::ppc_vsx_lxvd2x_be: 16391 case Intrinsic::ppc_vsx_lxvw4x_be: 16392 case Intrinsic::ppc_vsx_lxvl: 16393 case Intrinsic::ppc_vsx_lxvll: { 16394 EVT VT; 16395 switch (Intrinsic) { 16396 case Intrinsic::ppc_altivec_lvebx: 16397 VT = MVT::i8; 16398 break; 16399 case Intrinsic::ppc_altivec_lvehx: 16400 VT = MVT::i16; 16401 break; 16402 case Intrinsic::ppc_altivec_lvewx: 16403 VT = MVT::i32; 16404 break; 16405 case Intrinsic::ppc_vsx_lxvd2x: 16406 case Intrinsic::ppc_vsx_lxvd2x_be: 16407 VT = MVT::v2f64; 16408 break; 16409 default: 16410 VT = MVT::v4i32; 16411 break; 16412 } 16413 16414 Info.opc = ISD::INTRINSIC_W_CHAIN; 16415 Info.memVT = VT; 16416 Info.ptrVal = I.getArgOperand(0); 16417 Info.offset = -VT.getStoreSize()+1; 16418 Info.size = 2*VT.getStoreSize()-1; 16419 Info.align = Align(1); 16420 Info.flags = MachineMemOperand::MOLoad; 16421 return true; 16422 } 16423 case Intrinsic::ppc_altivec_stvx: 16424 case Intrinsic::ppc_altivec_stvxl: 16425 case Intrinsic::ppc_altivec_stvebx: 16426 case Intrinsic::ppc_altivec_stvehx: 16427 case Intrinsic::ppc_altivec_stvewx: 16428 case Intrinsic::ppc_vsx_stxvd2x: 16429 case Intrinsic::ppc_vsx_stxvw4x: 16430 case Intrinsic::ppc_vsx_stxvd2x_be: 16431 case Intrinsic::ppc_vsx_stxvw4x_be: 16432 case Intrinsic::ppc_vsx_stxvl: 16433 case Intrinsic::ppc_vsx_stxvll: { 16434 EVT VT; 16435 switch (Intrinsic) { 16436 case Intrinsic::ppc_altivec_stvebx: 16437 VT = MVT::i8; 16438 break; 16439 case Intrinsic::ppc_altivec_stvehx: 16440 VT = MVT::i16; 16441 break; 16442 case Intrinsic::ppc_altivec_stvewx: 16443 VT = MVT::i32; 16444 break; 16445 case Intrinsic::ppc_vsx_stxvd2x: 16446 case Intrinsic::ppc_vsx_stxvd2x_be: 16447 VT = MVT::v2f64; 16448 break; 16449 default: 16450 VT = MVT::v4i32; 16451 break; 16452 } 16453 16454 Info.opc = ISD::INTRINSIC_VOID; 16455 Info.memVT = VT; 16456 Info.ptrVal = I.getArgOperand(1); 16457 Info.offset = -VT.getStoreSize()+1; 16458 Info.size = 2*VT.getStoreSize()-1; 16459 Info.align = Align(1); 16460 Info.flags = MachineMemOperand::MOStore; 16461 return true; 16462 } 16463 default: 16464 break; 16465 } 16466 16467 return false; 16468 } 16469 16470 /// It returns EVT::Other if the type should be determined using generic 16471 /// target-independent logic. 16472 EVT PPCTargetLowering::getOptimalMemOpType( 16473 const MemOp &Op, const AttributeList &FuncAttributes) const { 16474 if (getTargetMachine().getOptLevel() != CodeGenOpt::None) { 16475 // We should use Altivec/VSX loads and stores when available. For unaligned 16476 // addresses, unaligned VSX loads are only fast starting with the P8. 16477 if (Subtarget.hasAltivec() && Op.size() >= 16 && 16478 (Op.isAligned(Align(16)) || 16479 ((Op.isMemset() && Subtarget.hasVSX()) || Subtarget.hasP8Vector()))) 16480 return MVT::v4i32; 16481 } 16482 16483 if (Subtarget.isPPC64()) { 16484 return MVT::i64; 16485 } 16486 16487 return MVT::i32; 16488 } 16489 16490 /// Returns true if it is beneficial to convert a load of a constant 16491 /// to just the constant itself. 16492 bool PPCTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm, 16493 Type *Ty) const { 16494 assert(Ty->isIntegerTy()); 16495 16496 unsigned BitSize = Ty->getPrimitiveSizeInBits(); 16497 return !(BitSize == 0 || BitSize > 64); 16498 } 16499 16500 bool PPCTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const { 16501 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy()) 16502 return false; 16503 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); 16504 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); 16505 return NumBits1 == 64 && NumBits2 == 32; 16506 } 16507 16508 bool PPCTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const { 16509 if (!VT1.isInteger() || !VT2.isInteger()) 16510 return false; 16511 unsigned NumBits1 = VT1.getSizeInBits(); 16512 unsigned NumBits2 = VT2.getSizeInBits(); 16513 return NumBits1 == 64 && NumBits2 == 32; 16514 } 16515 16516 bool PPCTargetLowering::isZExtFree(SDValue Val, EVT VT2) const { 16517 // Generally speaking, zexts are not free, but they are free when they can be 16518 // folded with other operations. 16519 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val)) { 16520 EVT MemVT = LD->getMemoryVT(); 16521 if ((MemVT == MVT::i1 || MemVT == MVT::i8 || MemVT == MVT::i16 || 16522 (Subtarget.isPPC64() && MemVT == MVT::i32)) && 16523 (LD->getExtensionType() == ISD::NON_EXTLOAD || 16524 LD->getExtensionType() == ISD::ZEXTLOAD)) 16525 return true; 16526 } 16527 16528 // FIXME: Add other cases... 16529 // - 32-bit shifts with a zext to i64 16530 // - zext after ctlz, bswap, etc. 16531 // - zext after and by a constant mask 16532 16533 return TargetLowering::isZExtFree(Val, VT2); 16534 } 16535 16536 bool PPCTargetLowering::isFPExtFree(EVT DestVT, EVT SrcVT) const { 16537 assert(DestVT.isFloatingPoint() && SrcVT.isFloatingPoint() && 16538 "invalid fpext types"); 16539 // Extending to float128 is not free. 16540 if (DestVT == MVT::f128) 16541 return false; 16542 return true; 16543 } 16544 16545 bool PPCTargetLowering::isLegalICmpImmediate(int64_t Imm) const { 16546 return isInt<16>(Imm) || isUInt<16>(Imm); 16547 } 16548 16549 bool PPCTargetLowering::isLegalAddImmediate(int64_t Imm) const { 16550 return isInt<16>(Imm) || isUInt<16>(Imm); 16551 } 16552 16553 bool PPCTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned, Align, 16554 MachineMemOperand::Flags, 16555 bool *Fast) const { 16556 if (DisablePPCUnaligned) 16557 return false; 16558 16559 // PowerPC supports unaligned memory access for simple non-vector types. 16560 // Although accessing unaligned addresses is not as efficient as accessing 16561 // aligned addresses, it is generally more efficient than manual expansion, 16562 // and generally only traps for software emulation when crossing page 16563 // boundaries. 16564 16565 if (!VT.isSimple()) 16566 return false; 16567 16568 if (VT.isFloatingPoint() && !VT.isVector() && 16569 !Subtarget.allowsUnalignedFPAccess()) 16570 return false; 16571 16572 if (VT.getSimpleVT().isVector()) { 16573 if (Subtarget.hasVSX()) { 16574 if (VT != MVT::v2f64 && VT != MVT::v2i64 && 16575 VT != MVT::v4f32 && VT != MVT::v4i32) 16576 return false; 16577 } else { 16578 return false; 16579 } 16580 } 16581 16582 if (VT == MVT::ppcf128) 16583 return false; 16584 16585 if (Fast) 16586 *Fast = true; 16587 16588 return true; 16589 } 16590 16591 bool PPCTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT, 16592 SDValue C) const { 16593 // Check integral scalar types. 16594 if (!VT.isScalarInteger()) 16595 return false; 16596 if (auto *ConstNode = dyn_cast<ConstantSDNode>(C.getNode())) { 16597 if (!ConstNode->getAPIntValue().isSignedIntN(64)) 16598 return false; 16599 // This transformation will generate >= 2 operations. But the following 16600 // cases will generate <= 2 instructions during ISEL. So exclude them. 16601 // 1. If the constant multiplier fits 16 bits, it can be handled by one 16602 // HW instruction, ie. MULLI 16603 // 2. If the multiplier after shifted fits 16 bits, an extra shift 16604 // instruction is needed than case 1, ie. MULLI and RLDICR 16605 int64_t Imm = ConstNode->getSExtValue(); 16606 unsigned Shift = countTrailingZeros<uint64_t>(Imm); 16607 Imm >>= Shift; 16608 if (isInt<16>(Imm)) 16609 return false; 16610 uint64_t UImm = static_cast<uint64_t>(Imm); 16611 if (isPowerOf2_64(UImm + 1) || isPowerOf2_64(UImm - 1) || 16612 isPowerOf2_64(1 - UImm) || isPowerOf2_64(-1 - UImm)) 16613 return true; 16614 } 16615 return false; 16616 } 16617 16618 bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, 16619 EVT VT) const { 16620 return isFMAFasterThanFMulAndFAdd( 16621 MF.getFunction(), VT.getTypeForEVT(MF.getFunction().getContext())); 16622 } 16623 16624 bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F, 16625 Type *Ty) const { 16626 switch (Ty->getScalarType()->getTypeID()) { 16627 case Type::FloatTyID: 16628 case Type::DoubleTyID: 16629 return true; 16630 case Type::FP128TyID: 16631 return Subtarget.hasP9Vector(); 16632 default: 16633 return false; 16634 } 16635 } 16636 16637 // FIXME: add more patterns which are not profitable to hoist. 16638 bool PPCTargetLowering::isProfitableToHoist(Instruction *I) const { 16639 if (!I->hasOneUse()) 16640 return true; 16641 16642 Instruction *User = I->user_back(); 16643 assert(User && "A single use instruction with no uses."); 16644 16645 switch (I->getOpcode()) { 16646 case Instruction::FMul: { 16647 // Don't break FMA, PowerPC prefers FMA. 16648 if (User->getOpcode() != Instruction::FSub && 16649 User->getOpcode() != Instruction::FAdd) 16650 return true; 16651 16652 const TargetOptions &Options = getTargetMachine().Options; 16653 const Function *F = I->getFunction(); 16654 const DataLayout &DL = F->getParent()->getDataLayout(); 16655 Type *Ty = User->getOperand(0)->getType(); 16656 16657 return !( 16658 isFMAFasterThanFMulAndFAdd(*F, Ty) && 16659 isOperationLegalOrCustom(ISD::FMA, getValueType(DL, Ty)) && 16660 (Options.AllowFPOpFusion == FPOpFusion::Fast || Options.UnsafeFPMath)); 16661 } 16662 case Instruction::Load: { 16663 // Don't break "store (load float*)" pattern, this pattern will be combined 16664 // to "store (load int32)" in later InstCombine pass. See function 16665 // combineLoadToOperationType. On PowerPC, loading a float point takes more 16666 // cycles than loading a 32 bit integer. 16667 LoadInst *LI = cast<LoadInst>(I); 16668 // For the loads that combineLoadToOperationType does nothing, like 16669 // ordered load, it should be profitable to hoist them. 16670 // For swifterror load, it can only be used for pointer to pointer type, so 16671 // later type check should get rid of this case. 16672 if (!LI->isUnordered()) 16673 return true; 16674 16675 if (User->getOpcode() != Instruction::Store) 16676 return true; 16677 16678 if (I->getType()->getTypeID() != Type::FloatTyID) 16679 return true; 16680 16681 return false; 16682 } 16683 default: 16684 return true; 16685 } 16686 return true; 16687 } 16688 16689 const MCPhysReg * 16690 PPCTargetLowering::getScratchRegisters(CallingConv::ID) const { 16691 // LR is a callee-save register, but we must treat it as clobbered by any call 16692 // site. Hence we include LR in the scratch registers, which are in turn added 16693 // as implicit-defs for stackmaps and patchpoints. The same reasoning applies 16694 // to CTR, which is used by any indirect call. 16695 static const MCPhysReg ScratchRegs[] = { 16696 PPC::X12, PPC::LR8, PPC::CTR8, 0 16697 }; 16698 16699 return ScratchRegs; 16700 } 16701 16702 Register PPCTargetLowering::getExceptionPointerRegister( 16703 const Constant *PersonalityFn) const { 16704 return Subtarget.isPPC64() ? PPC::X3 : PPC::R3; 16705 } 16706 16707 Register PPCTargetLowering::getExceptionSelectorRegister( 16708 const Constant *PersonalityFn) const { 16709 return Subtarget.isPPC64() ? PPC::X4 : PPC::R4; 16710 } 16711 16712 bool 16713 PPCTargetLowering::shouldExpandBuildVectorWithShuffles( 16714 EVT VT , unsigned DefinedValues) const { 16715 if (VT == MVT::v2i64) 16716 return Subtarget.hasDirectMove(); // Don't need stack ops with direct moves 16717 16718 if (Subtarget.hasVSX()) 16719 return true; 16720 16721 return TargetLowering::shouldExpandBuildVectorWithShuffles(VT, DefinedValues); 16722 } 16723 16724 Sched::Preference PPCTargetLowering::getSchedulingPreference(SDNode *N) const { 16725 if (DisableILPPref || Subtarget.enableMachineScheduler()) 16726 return TargetLowering::getSchedulingPreference(N); 16727 16728 return Sched::ILP; 16729 } 16730 16731 // Create a fast isel object. 16732 FastISel * 16733 PPCTargetLowering::createFastISel(FunctionLoweringInfo &FuncInfo, 16734 const TargetLibraryInfo *LibInfo) const { 16735 return PPC::createFastISel(FuncInfo, LibInfo); 16736 } 16737 16738 // 'Inverted' means the FMA opcode after negating one multiplicand. 16739 // For example, (fma -a b c) = (fnmsub a b c) 16740 static unsigned invertFMAOpcode(unsigned Opc) { 16741 switch (Opc) { 16742 default: 16743 llvm_unreachable("Invalid FMA opcode for PowerPC!"); 16744 case ISD::FMA: 16745 return PPCISD::FNMSUB; 16746 case PPCISD::FNMSUB: 16747 return ISD::FMA; 16748 } 16749 } 16750 16751 SDValue PPCTargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG, 16752 bool LegalOps, bool OptForSize, 16753 NegatibleCost &Cost, 16754 unsigned Depth) const { 16755 if (Depth > SelectionDAG::MaxRecursionDepth) 16756 return SDValue(); 16757 16758 unsigned Opc = Op.getOpcode(); 16759 EVT VT = Op.getValueType(); 16760 SDNodeFlags Flags = Op.getNode()->getFlags(); 16761 16762 switch (Opc) { 16763 case PPCISD::FNMSUB: 16764 if (!Op.hasOneUse() || !isTypeLegal(VT)) 16765 break; 16766 16767 const TargetOptions &Options = getTargetMachine().Options; 16768 SDValue N0 = Op.getOperand(0); 16769 SDValue N1 = Op.getOperand(1); 16770 SDValue N2 = Op.getOperand(2); 16771 SDLoc Loc(Op); 16772 16773 NegatibleCost N2Cost = NegatibleCost::Expensive; 16774 SDValue NegN2 = 16775 getNegatedExpression(N2, DAG, LegalOps, OptForSize, N2Cost, Depth + 1); 16776 16777 if (!NegN2) 16778 return SDValue(); 16779 16780 // (fneg (fnmsub a b c)) => (fnmsub (fneg a) b (fneg c)) 16781 // (fneg (fnmsub a b c)) => (fnmsub a (fneg b) (fneg c)) 16782 // These transformations may change sign of zeroes. For example, 16783 // -(-ab-(-c))=-0 while -(-(ab-c))=+0 when a=b=c=1. 16784 if (Flags.hasNoSignedZeros() || Options.NoSignedZerosFPMath) { 16785 // Try and choose the cheaper one to negate. 16786 NegatibleCost N0Cost = NegatibleCost::Expensive; 16787 SDValue NegN0 = getNegatedExpression(N0, DAG, LegalOps, OptForSize, 16788 N0Cost, Depth + 1); 16789 16790 NegatibleCost N1Cost = NegatibleCost::Expensive; 16791 SDValue NegN1 = getNegatedExpression(N1, DAG, LegalOps, OptForSize, 16792 N1Cost, Depth + 1); 16793 16794 if (NegN0 && N0Cost <= N1Cost) { 16795 Cost = std::min(N0Cost, N2Cost); 16796 return DAG.getNode(Opc, Loc, VT, NegN0, N1, NegN2, Flags); 16797 } else if (NegN1) { 16798 Cost = std::min(N1Cost, N2Cost); 16799 return DAG.getNode(Opc, Loc, VT, N0, NegN1, NegN2, Flags); 16800 } 16801 } 16802 16803 // (fneg (fnmsub a b c)) => (fma a b (fneg c)) 16804 if (isOperationLegal(ISD::FMA, VT)) { 16805 Cost = N2Cost; 16806 return DAG.getNode(ISD::FMA, Loc, VT, N0, N1, NegN2, Flags); 16807 } 16808 16809 break; 16810 } 16811 16812 return TargetLowering::getNegatedExpression(Op, DAG, LegalOps, OptForSize, 16813 Cost, Depth); 16814 } 16815 16816 // Override to enable LOAD_STACK_GUARD lowering on Linux. 16817 bool PPCTargetLowering::useLoadStackGuardNode() const { 16818 if (!Subtarget.isTargetLinux()) 16819 return TargetLowering::useLoadStackGuardNode(); 16820 return true; 16821 } 16822 16823 // Override to disable global variable loading on Linux and insert AIX canary 16824 // word declaration. 16825 void PPCTargetLowering::insertSSPDeclarations(Module &M) const { 16826 if (Subtarget.isAIXABI()) { 16827 M.getOrInsertGlobal(AIXSSPCanaryWordName, 16828 Type::getInt8PtrTy(M.getContext())); 16829 return; 16830 } 16831 if (!Subtarget.isTargetLinux()) 16832 return TargetLowering::insertSSPDeclarations(M); 16833 } 16834 16835 Value *PPCTargetLowering::getSDagStackGuard(const Module &M) const { 16836 if (Subtarget.isAIXABI()) 16837 return M.getGlobalVariable(AIXSSPCanaryWordName); 16838 return TargetLowering::getSDagStackGuard(M); 16839 } 16840 16841 bool PPCTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, 16842 bool ForCodeSize) const { 16843 if (!VT.isSimple() || !Subtarget.hasVSX()) 16844 return false; 16845 16846 switch(VT.getSimpleVT().SimpleTy) { 16847 default: 16848 // For FP types that are currently not supported by PPC backend, return 16849 // false. Examples: f16, f80. 16850 return false; 16851 case MVT::f32: 16852 case MVT::f64: 16853 if (Subtarget.hasPrefixInstrs()) { 16854 // we can materialize all immediatess via XXSPLTI32DX and XXSPLTIDP. 16855 return true; 16856 } 16857 LLVM_FALLTHROUGH; 16858 case MVT::ppcf128: 16859 return Imm.isPosZero(); 16860 } 16861 } 16862 16863 // For vector shift operation op, fold 16864 // (op x, (and y, ((1 << numbits(x)) - 1))) -> (target op x, y) 16865 static SDValue stripModuloOnShift(const TargetLowering &TLI, SDNode *N, 16866 SelectionDAG &DAG) { 16867 SDValue N0 = N->getOperand(0); 16868 SDValue N1 = N->getOperand(1); 16869 EVT VT = N0.getValueType(); 16870 unsigned OpSizeInBits = VT.getScalarSizeInBits(); 16871 unsigned Opcode = N->getOpcode(); 16872 unsigned TargetOpcode; 16873 16874 switch (Opcode) { 16875 default: 16876 llvm_unreachable("Unexpected shift operation"); 16877 case ISD::SHL: 16878 TargetOpcode = PPCISD::SHL; 16879 break; 16880 case ISD::SRL: 16881 TargetOpcode = PPCISD::SRL; 16882 break; 16883 case ISD::SRA: 16884 TargetOpcode = PPCISD::SRA; 16885 break; 16886 } 16887 16888 if (VT.isVector() && TLI.isOperationLegal(Opcode, VT) && 16889 N1->getOpcode() == ISD::AND) 16890 if (ConstantSDNode *Mask = isConstOrConstSplat(N1->getOperand(1))) 16891 if (Mask->getZExtValue() == OpSizeInBits - 1) 16892 return DAG.getNode(TargetOpcode, SDLoc(N), VT, N0, N1->getOperand(0)); 16893 16894 return SDValue(); 16895 } 16896 16897 SDValue PPCTargetLowering::combineSHL(SDNode *N, DAGCombinerInfo &DCI) const { 16898 if (auto Value = stripModuloOnShift(*this, N, DCI.DAG)) 16899 return Value; 16900 16901 SDValue N0 = N->getOperand(0); 16902 ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(N->getOperand(1)); 16903 if (!Subtarget.isISA3_0() || !Subtarget.isPPC64() || 16904 N0.getOpcode() != ISD::SIGN_EXTEND || 16905 N0.getOperand(0).getValueType() != MVT::i32 || CN1 == nullptr || 16906 N->getValueType(0) != MVT::i64) 16907 return SDValue(); 16908 16909 // We can't save an operation here if the value is already extended, and 16910 // the existing shift is easier to combine. 16911 SDValue ExtsSrc = N0.getOperand(0); 16912 if (ExtsSrc.getOpcode() == ISD::TRUNCATE && 16913 ExtsSrc.getOperand(0).getOpcode() == ISD::AssertSext) 16914 return SDValue(); 16915 16916 SDLoc DL(N0); 16917 SDValue ShiftBy = SDValue(CN1, 0); 16918 // We want the shift amount to be i32 on the extswli, but the shift could 16919 // have an i64. 16920 if (ShiftBy.getValueType() == MVT::i64) 16921 ShiftBy = DCI.DAG.getConstant(CN1->getZExtValue(), DL, MVT::i32); 16922 16923 return DCI.DAG.getNode(PPCISD::EXTSWSLI, DL, MVT::i64, N0->getOperand(0), 16924 ShiftBy); 16925 } 16926 16927 SDValue PPCTargetLowering::combineSRA(SDNode *N, DAGCombinerInfo &DCI) const { 16928 if (auto Value = stripModuloOnShift(*this, N, DCI.DAG)) 16929 return Value; 16930 16931 return SDValue(); 16932 } 16933 16934 SDValue PPCTargetLowering::combineSRL(SDNode *N, DAGCombinerInfo &DCI) const { 16935 if (auto Value = stripModuloOnShift(*this, N, DCI.DAG)) 16936 return Value; 16937 16938 return SDValue(); 16939 } 16940 16941 // Transform (add X, (zext(setne Z, C))) -> (addze X, (addic (addi Z, -C), -1)) 16942 // Transform (add X, (zext(sete Z, C))) -> (addze X, (subfic (addi Z, -C), 0)) 16943 // When C is zero, the equation (addi Z, -C) can be simplified to Z 16944 // Requirement: -C in [-32768, 32767], X and Z are MVT::i64 types 16945 static SDValue combineADDToADDZE(SDNode *N, SelectionDAG &DAG, 16946 const PPCSubtarget &Subtarget) { 16947 if (!Subtarget.isPPC64()) 16948 return SDValue(); 16949 16950 SDValue LHS = N->getOperand(0); 16951 SDValue RHS = N->getOperand(1); 16952 16953 auto isZextOfCompareWithConstant = [](SDValue Op) { 16954 if (Op.getOpcode() != ISD::ZERO_EXTEND || !Op.hasOneUse() || 16955 Op.getValueType() != MVT::i64) 16956 return false; 16957 16958 SDValue Cmp = Op.getOperand(0); 16959 if (Cmp.getOpcode() != ISD::SETCC || !Cmp.hasOneUse() || 16960 Cmp.getOperand(0).getValueType() != MVT::i64) 16961 return false; 16962 16963 if (auto *Constant = dyn_cast<ConstantSDNode>(Cmp.getOperand(1))) { 16964 int64_t NegConstant = 0 - Constant->getSExtValue(); 16965 // Due to the limitations of the addi instruction, 16966 // -C is required to be [-32768, 32767]. 16967 return isInt<16>(NegConstant); 16968 } 16969 16970 return false; 16971 }; 16972 16973 bool LHSHasPattern = isZextOfCompareWithConstant(LHS); 16974 bool RHSHasPattern = isZextOfCompareWithConstant(RHS); 16975 16976 // If there is a pattern, canonicalize a zext operand to the RHS. 16977 if (LHSHasPattern && !RHSHasPattern) 16978 std::swap(LHS, RHS); 16979 else if (!LHSHasPattern && !RHSHasPattern) 16980 return SDValue(); 16981 16982 SDLoc DL(N); 16983 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Glue); 16984 SDValue Cmp = RHS.getOperand(0); 16985 SDValue Z = Cmp.getOperand(0); 16986 auto *Constant = cast<ConstantSDNode>(Cmp.getOperand(1)); 16987 int64_t NegConstant = 0 - Constant->getSExtValue(); 16988 16989 switch(cast<CondCodeSDNode>(Cmp.getOperand(2))->get()) { 16990 default: break; 16991 case ISD::SETNE: { 16992 // when C == 0 16993 // --> addze X, (addic Z, -1).carry 16994 // / 16995 // add X, (zext(setne Z, C))-- 16996 // \ when -32768 <= -C <= 32767 && C != 0 16997 // --> addze X, (addic (addi Z, -C), -1).carry 16998 SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Z, 16999 DAG.getConstant(NegConstant, DL, MVT::i64)); 17000 SDValue AddOrZ = NegConstant != 0 ? Add : Z; 17001 SDValue Addc = DAG.getNode(ISD::ADDC, DL, DAG.getVTList(MVT::i64, MVT::Glue), 17002 AddOrZ, DAG.getConstant(-1ULL, DL, MVT::i64)); 17003 return DAG.getNode(ISD::ADDE, DL, VTs, LHS, DAG.getConstant(0, DL, MVT::i64), 17004 SDValue(Addc.getNode(), 1)); 17005 } 17006 case ISD::SETEQ: { 17007 // when C == 0 17008 // --> addze X, (subfic Z, 0).carry 17009 // / 17010 // add X, (zext(sete Z, C))-- 17011 // \ when -32768 <= -C <= 32767 && C != 0 17012 // --> addze X, (subfic (addi Z, -C), 0).carry 17013 SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Z, 17014 DAG.getConstant(NegConstant, DL, MVT::i64)); 17015 SDValue AddOrZ = NegConstant != 0 ? Add : Z; 17016 SDValue Subc = DAG.getNode(ISD::SUBC, DL, DAG.getVTList(MVT::i64, MVT::Glue), 17017 DAG.getConstant(0, DL, MVT::i64), AddOrZ); 17018 return DAG.getNode(ISD::ADDE, DL, VTs, LHS, DAG.getConstant(0, DL, MVT::i64), 17019 SDValue(Subc.getNode(), 1)); 17020 } 17021 } 17022 17023 return SDValue(); 17024 } 17025 17026 // Transform 17027 // (add C1, (MAT_PCREL_ADDR GlobalAddr+C2)) to 17028 // (MAT_PCREL_ADDR GlobalAddr+(C1+C2)) 17029 // In this case both C1 and C2 must be known constants. 17030 // C1+C2 must fit into a 34 bit signed integer. 17031 static SDValue combineADDToMAT_PCREL_ADDR(SDNode *N, SelectionDAG &DAG, 17032 const PPCSubtarget &Subtarget) { 17033 if (!Subtarget.isUsingPCRelativeCalls()) 17034 return SDValue(); 17035 17036 // Check both Operand 0 and Operand 1 of the ADD node for the PCRel node. 17037 // If we find that node try to cast the Global Address and the Constant. 17038 SDValue LHS = N->getOperand(0); 17039 SDValue RHS = N->getOperand(1); 17040 17041 if (LHS.getOpcode() != PPCISD::MAT_PCREL_ADDR) 17042 std::swap(LHS, RHS); 17043 17044 if (LHS.getOpcode() != PPCISD::MAT_PCREL_ADDR) 17045 return SDValue(); 17046 17047 // Operand zero of PPCISD::MAT_PCREL_ADDR is the GA node. 17048 GlobalAddressSDNode *GSDN = dyn_cast<GlobalAddressSDNode>(LHS.getOperand(0)); 17049 ConstantSDNode* ConstNode = dyn_cast<ConstantSDNode>(RHS); 17050 17051 // Check that both casts succeeded. 17052 if (!GSDN || !ConstNode) 17053 return SDValue(); 17054 17055 int64_t NewOffset = GSDN->getOffset() + ConstNode->getSExtValue(); 17056 SDLoc DL(GSDN); 17057 17058 // The signed int offset needs to fit in 34 bits. 17059 if (!isInt<34>(NewOffset)) 17060 return SDValue(); 17061 17062 // The new global address is a copy of the old global address except 17063 // that it has the updated Offset. 17064 SDValue GA = 17065 DAG.getTargetGlobalAddress(GSDN->getGlobal(), DL, GSDN->getValueType(0), 17066 NewOffset, GSDN->getTargetFlags()); 17067 SDValue MatPCRel = 17068 DAG.getNode(PPCISD::MAT_PCREL_ADDR, DL, GSDN->getValueType(0), GA); 17069 return MatPCRel; 17070 } 17071 17072 SDValue PPCTargetLowering::combineADD(SDNode *N, DAGCombinerInfo &DCI) const { 17073 if (auto Value = combineADDToADDZE(N, DCI.DAG, Subtarget)) 17074 return Value; 17075 17076 if (auto Value = combineADDToMAT_PCREL_ADDR(N, DCI.DAG, Subtarget)) 17077 return Value; 17078 17079 return SDValue(); 17080 } 17081 17082 // Detect TRUNCATE operations on bitcasts of float128 values. 17083 // What we are looking for here is the situtation where we extract a subset 17084 // of bits from a 128 bit float. 17085 // This can be of two forms: 17086 // 1) BITCAST of f128 feeding TRUNCATE 17087 // 2) BITCAST of f128 feeding SRL (a shift) feeding TRUNCATE 17088 // The reason this is required is because we do not have a legal i128 type 17089 // and so we want to prevent having to store the f128 and then reload part 17090 // of it. 17091 SDValue PPCTargetLowering::combineTRUNCATE(SDNode *N, 17092 DAGCombinerInfo &DCI) const { 17093 // If we are using CRBits then try that first. 17094 if (Subtarget.useCRBits()) { 17095 // Check if CRBits did anything and return that if it did. 17096 if (SDValue CRTruncValue = DAGCombineTruncBoolExt(N, DCI)) 17097 return CRTruncValue; 17098 } 17099 17100 SDLoc dl(N); 17101 SDValue Op0 = N->getOperand(0); 17102 17103 // fold (truncate (abs (sub (zext a), (zext b)))) -> (vabsd a, b) 17104 if (Subtarget.hasP9Altivec() && Op0.getOpcode() == ISD::ABS) { 17105 EVT VT = N->getValueType(0); 17106 if (VT != MVT::v4i32 && VT != MVT::v8i16 && VT != MVT::v16i8) 17107 return SDValue(); 17108 SDValue Sub = Op0.getOperand(0); 17109 if (Sub.getOpcode() == ISD::SUB) { 17110 SDValue SubOp0 = Sub.getOperand(0); 17111 SDValue SubOp1 = Sub.getOperand(1); 17112 if ((SubOp0.getOpcode() == ISD::ZERO_EXTEND) && 17113 (SubOp1.getOpcode() == ISD::ZERO_EXTEND)) { 17114 return DCI.DAG.getNode(PPCISD::VABSD, dl, VT, SubOp0.getOperand(0), 17115 SubOp1.getOperand(0), 17116 DCI.DAG.getTargetConstant(0, dl, MVT::i32)); 17117 } 17118 } 17119 } 17120 17121 // Looking for a truncate of i128 to i64. 17122 if (Op0.getValueType() != MVT::i128 || N->getValueType(0) != MVT::i64) 17123 return SDValue(); 17124 17125 int EltToExtract = DCI.DAG.getDataLayout().isBigEndian() ? 1 : 0; 17126 17127 // SRL feeding TRUNCATE. 17128 if (Op0.getOpcode() == ISD::SRL) { 17129 ConstantSDNode *ConstNode = dyn_cast<ConstantSDNode>(Op0.getOperand(1)); 17130 // The right shift has to be by 64 bits. 17131 if (!ConstNode || ConstNode->getZExtValue() != 64) 17132 return SDValue(); 17133 17134 // Switch the element number to extract. 17135 EltToExtract = EltToExtract ? 0 : 1; 17136 // Update Op0 past the SRL. 17137 Op0 = Op0.getOperand(0); 17138 } 17139 17140 // BITCAST feeding a TRUNCATE possibly via SRL. 17141 if (Op0.getOpcode() == ISD::BITCAST && 17142 Op0.getValueType() == MVT::i128 && 17143 Op0.getOperand(0).getValueType() == MVT::f128) { 17144 SDValue Bitcast = DCI.DAG.getBitcast(MVT::v2i64, Op0.getOperand(0)); 17145 return DCI.DAG.getNode( 17146 ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Bitcast, 17147 DCI.DAG.getTargetConstant(EltToExtract, dl, MVT::i32)); 17148 } 17149 return SDValue(); 17150 } 17151 17152 SDValue PPCTargetLowering::combineMUL(SDNode *N, DAGCombinerInfo &DCI) const { 17153 SelectionDAG &DAG = DCI.DAG; 17154 17155 ConstantSDNode *ConstOpOrElement = isConstOrConstSplat(N->getOperand(1)); 17156 if (!ConstOpOrElement) 17157 return SDValue(); 17158 17159 // An imul is usually smaller than the alternative sequence for legal type. 17160 if (DAG.getMachineFunction().getFunction().hasMinSize() && 17161 isOperationLegal(ISD::MUL, N->getValueType(0))) 17162 return SDValue(); 17163 17164 auto IsProfitable = [this](bool IsNeg, bool IsAddOne, EVT VT) -> bool { 17165 switch (this->Subtarget.getCPUDirective()) { 17166 default: 17167 // TODO: enhance the condition for subtarget before pwr8 17168 return false; 17169 case PPC::DIR_PWR8: 17170 // type mul add shl 17171 // scalar 4 1 1 17172 // vector 7 2 2 17173 return true; 17174 case PPC::DIR_PWR9: 17175 case PPC::DIR_PWR10: 17176 case PPC::DIR_PWR_FUTURE: 17177 // type mul add shl 17178 // scalar 5 2 2 17179 // vector 7 2 2 17180 17181 // The cycle RATIO of related operations are showed as a table above. 17182 // Because mul is 5(scalar)/7(vector), add/sub/shl are all 2 for both 17183 // scalar and vector type. For 2 instrs patterns, add/sub + shl 17184 // are 4, it is always profitable; but for 3 instrs patterns 17185 // (mul x, -(2^N + 1)) => -(add (shl x, N), x), sub + add + shl are 6. 17186 // So we should only do it for vector type. 17187 return IsAddOne && IsNeg ? VT.isVector() : true; 17188 } 17189 }; 17190 17191 EVT VT = N->getValueType(0); 17192 SDLoc DL(N); 17193 17194 const APInt &MulAmt = ConstOpOrElement->getAPIntValue(); 17195 bool IsNeg = MulAmt.isNegative(); 17196 APInt MulAmtAbs = MulAmt.abs(); 17197 17198 if ((MulAmtAbs - 1).isPowerOf2()) { 17199 // (mul x, 2^N + 1) => (add (shl x, N), x) 17200 // (mul x, -(2^N + 1)) => -(add (shl x, N), x) 17201 17202 if (!IsProfitable(IsNeg, true, VT)) 17203 return SDValue(); 17204 17205 SDValue Op0 = N->getOperand(0); 17206 SDValue Op1 = 17207 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), 17208 DAG.getConstant((MulAmtAbs - 1).logBase2(), DL, VT)); 17209 SDValue Res = DAG.getNode(ISD::ADD, DL, VT, Op0, Op1); 17210 17211 if (!IsNeg) 17212 return Res; 17213 17214 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Res); 17215 } else if ((MulAmtAbs + 1).isPowerOf2()) { 17216 // (mul x, 2^N - 1) => (sub (shl x, N), x) 17217 // (mul x, -(2^N - 1)) => (sub x, (shl x, N)) 17218 17219 if (!IsProfitable(IsNeg, false, VT)) 17220 return SDValue(); 17221 17222 SDValue Op0 = N->getOperand(0); 17223 SDValue Op1 = 17224 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), 17225 DAG.getConstant((MulAmtAbs + 1).logBase2(), DL, VT)); 17226 17227 if (!IsNeg) 17228 return DAG.getNode(ISD::SUB, DL, VT, Op1, Op0); 17229 else 17230 return DAG.getNode(ISD::SUB, DL, VT, Op0, Op1); 17231 17232 } else { 17233 return SDValue(); 17234 } 17235 } 17236 17237 // Combine fma-like op (like fnmsub) with fnegs to appropriate op. Do this 17238 // in combiner since we need to check SD flags and other subtarget features. 17239 SDValue PPCTargetLowering::combineFMALike(SDNode *N, 17240 DAGCombinerInfo &DCI) const { 17241 SDValue N0 = N->getOperand(0); 17242 SDValue N1 = N->getOperand(1); 17243 SDValue N2 = N->getOperand(2); 17244 SDNodeFlags Flags = N->getFlags(); 17245 EVT VT = N->getValueType(0); 17246 SelectionDAG &DAG = DCI.DAG; 17247 const TargetOptions &Options = getTargetMachine().Options; 17248 unsigned Opc = N->getOpcode(); 17249 bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize(); 17250 bool LegalOps = !DCI.isBeforeLegalizeOps(); 17251 SDLoc Loc(N); 17252 17253 if (!isOperationLegal(ISD::FMA, VT)) 17254 return SDValue(); 17255 17256 // Allowing transformation to FNMSUB may change sign of zeroes when ab-c=0 17257 // since (fnmsub a b c)=-0 while c-ab=+0. 17258 if (!Flags.hasNoSignedZeros() && !Options.NoSignedZerosFPMath) 17259 return SDValue(); 17260 17261 // (fma (fneg a) b c) => (fnmsub a b c) 17262 // (fnmsub (fneg a) b c) => (fma a b c) 17263 if (SDValue NegN0 = getCheaperNegatedExpression(N0, DAG, LegalOps, CodeSize)) 17264 return DAG.getNode(invertFMAOpcode(Opc), Loc, VT, NegN0, N1, N2, Flags); 17265 17266 // (fma a (fneg b) c) => (fnmsub a b c) 17267 // (fnmsub a (fneg b) c) => (fma a b c) 17268 if (SDValue NegN1 = getCheaperNegatedExpression(N1, DAG, LegalOps, CodeSize)) 17269 return DAG.getNode(invertFMAOpcode(Opc), Loc, VT, N0, NegN1, N2, Flags); 17270 17271 return SDValue(); 17272 } 17273 17274 bool PPCTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const { 17275 // Only duplicate to increase tail-calls for the 64bit SysV ABIs. 17276 if (!Subtarget.is64BitELFABI()) 17277 return false; 17278 17279 // If not a tail call then no need to proceed. 17280 if (!CI->isTailCall()) 17281 return false; 17282 17283 // If sibling calls have been disabled and tail-calls aren't guaranteed 17284 // there is no reason to duplicate. 17285 auto &TM = getTargetMachine(); 17286 if (!TM.Options.GuaranteedTailCallOpt && DisableSCO) 17287 return false; 17288 17289 // Can't tail call a function called indirectly, or if it has variadic args. 17290 const Function *Callee = CI->getCalledFunction(); 17291 if (!Callee || Callee->isVarArg()) 17292 return false; 17293 17294 // Make sure the callee and caller calling conventions are eligible for tco. 17295 const Function *Caller = CI->getParent()->getParent(); 17296 if (!areCallingConvEligibleForTCO_64SVR4(Caller->getCallingConv(), 17297 CI->getCallingConv())) 17298 return false; 17299 17300 // If the function is local then we have a good chance at tail-calling it 17301 return getTargetMachine().shouldAssumeDSOLocal(*Caller->getParent(), Callee); 17302 } 17303 17304 bool PPCTargetLowering::hasBitPreservingFPLogic(EVT VT) const { 17305 if (!Subtarget.hasVSX()) 17306 return false; 17307 if (Subtarget.hasP9Vector() && VT == MVT::f128) 17308 return true; 17309 return VT == MVT::f32 || VT == MVT::f64 || 17310 VT == MVT::v4f32 || VT == MVT::v2f64; 17311 } 17312 17313 bool PPCTargetLowering:: 17314 isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const { 17315 const Value *Mask = AndI.getOperand(1); 17316 // If the mask is suitable for andi. or andis. we should sink the and. 17317 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Mask)) { 17318 // Can't handle constants wider than 64-bits. 17319 if (CI->getBitWidth() > 64) 17320 return false; 17321 int64_t ConstVal = CI->getZExtValue(); 17322 return isUInt<16>(ConstVal) || 17323 (isUInt<16>(ConstVal >> 16) && !(ConstVal & 0xFFFF)); 17324 } 17325 17326 // For non-constant masks, we can always use the record-form and. 17327 return true; 17328 } 17329 17330 // Transform (abs (sub (zext a), (zext b))) to (vabsd a b 0) 17331 // Transform (abs (sub (zext a), (zext_invec b))) to (vabsd a b 0) 17332 // Transform (abs (sub (zext_invec a), (zext_invec b))) to (vabsd a b 0) 17333 // Transform (abs (sub (zext_invec a), (zext b))) to (vabsd a b 0) 17334 // Transform (abs (sub a, b) to (vabsd a b 1)) if a & b of type v4i32 17335 SDValue PPCTargetLowering::combineABS(SDNode *N, DAGCombinerInfo &DCI) const { 17336 assert((N->getOpcode() == ISD::ABS) && "Need ABS node here"); 17337 assert(Subtarget.hasP9Altivec() && 17338 "Only combine this when P9 altivec supported!"); 17339 EVT VT = N->getValueType(0); 17340 if (VT != MVT::v4i32 && VT != MVT::v8i16 && VT != MVT::v16i8) 17341 return SDValue(); 17342 17343 SelectionDAG &DAG = DCI.DAG; 17344 SDLoc dl(N); 17345 if (N->getOperand(0).getOpcode() == ISD::SUB) { 17346 // Even for signed integers, if it's known to be positive (as signed 17347 // integer) due to zero-extended inputs. 17348 unsigned SubOpcd0 = N->getOperand(0)->getOperand(0).getOpcode(); 17349 unsigned SubOpcd1 = N->getOperand(0)->getOperand(1).getOpcode(); 17350 if ((SubOpcd0 == ISD::ZERO_EXTEND || 17351 SubOpcd0 == ISD::ZERO_EXTEND_VECTOR_INREG) && 17352 (SubOpcd1 == ISD::ZERO_EXTEND || 17353 SubOpcd1 == ISD::ZERO_EXTEND_VECTOR_INREG)) { 17354 return DAG.getNode(PPCISD::VABSD, dl, N->getOperand(0).getValueType(), 17355 N->getOperand(0)->getOperand(0), 17356 N->getOperand(0)->getOperand(1), 17357 DAG.getTargetConstant(0, dl, MVT::i32)); 17358 } 17359 17360 // For type v4i32, it can be optimized with xvnegsp + vabsduw 17361 if (N->getOperand(0).getValueType() == MVT::v4i32 && 17362 N->getOperand(0).hasOneUse()) { 17363 return DAG.getNode(PPCISD::VABSD, dl, N->getOperand(0).getValueType(), 17364 N->getOperand(0)->getOperand(0), 17365 N->getOperand(0)->getOperand(1), 17366 DAG.getTargetConstant(1, dl, MVT::i32)); 17367 } 17368 } 17369 17370 return SDValue(); 17371 } 17372 17373 // For type v4i32/v8ii16/v16i8, transform 17374 // from (vselect (setcc a, b, setugt), (sub a, b), (sub b, a)) to (vabsd a, b) 17375 // from (vselect (setcc a, b, setuge), (sub a, b), (sub b, a)) to (vabsd a, b) 17376 // from (vselect (setcc a, b, setult), (sub b, a), (sub a, b)) to (vabsd a, b) 17377 // from (vselect (setcc a, b, setule), (sub b, a), (sub a, b)) to (vabsd a, b) 17378 SDValue PPCTargetLowering::combineVSelect(SDNode *N, 17379 DAGCombinerInfo &DCI) const { 17380 assert((N->getOpcode() == ISD::VSELECT) && "Need VSELECT node here"); 17381 assert(Subtarget.hasP9Altivec() && 17382 "Only combine this when P9 altivec supported!"); 17383 17384 SelectionDAG &DAG = DCI.DAG; 17385 SDLoc dl(N); 17386 SDValue Cond = N->getOperand(0); 17387 SDValue TrueOpnd = N->getOperand(1); 17388 SDValue FalseOpnd = N->getOperand(2); 17389 EVT VT = N->getOperand(1).getValueType(); 17390 17391 if (Cond.getOpcode() != ISD::SETCC || TrueOpnd.getOpcode() != ISD::SUB || 17392 FalseOpnd.getOpcode() != ISD::SUB) 17393 return SDValue(); 17394 17395 // ABSD only available for type v4i32/v8i16/v16i8 17396 if (VT != MVT::v4i32 && VT != MVT::v8i16 && VT != MVT::v16i8) 17397 return SDValue(); 17398 17399 // At least to save one more dependent computation 17400 if (!(Cond.hasOneUse() || TrueOpnd.hasOneUse() || FalseOpnd.hasOneUse())) 17401 return SDValue(); 17402 17403 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); 17404 17405 // Can only handle unsigned comparison here 17406 switch (CC) { 17407 default: 17408 return SDValue(); 17409 case ISD::SETUGT: 17410 case ISD::SETUGE: 17411 break; 17412 case ISD::SETULT: 17413 case ISD::SETULE: 17414 std::swap(TrueOpnd, FalseOpnd); 17415 break; 17416 } 17417 17418 SDValue CmpOpnd1 = Cond.getOperand(0); 17419 SDValue CmpOpnd2 = Cond.getOperand(1); 17420 17421 // SETCC CmpOpnd1 CmpOpnd2 cond 17422 // TrueOpnd = CmpOpnd1 - CmpOpnd2 17423 // FalseOpnd = CmpOpnd2 - CmpOpnd1 17424 if (TrueOpnd.getOperand(0) == CmpOpnd1 && 17425 TrueOpnd.getOperand(1) == CmpOpnd2 && 17426 FalseOpnd.getOperand(0) == CmpOpnd2 && 17427 FalseOpnd.getOperand(1) == CmpOpnd1) { 17428 return DAG.getNode(PPCISD::VABSD, dl, N->getOperand(1).getValueType(), 17429 CmpOpnd1, CmpOpnd2, 17430 DAG.getTargetConstant(0, dl, MVT::i32)); 17431 } 17432 17433 return SDValue(); 17434 } 17435 17436 /// getAddrModeForFlags - Based on the set of address flags, select the most 17437 /// optimal instruction format to match by. 17438 PPC::AddrMode PPCTargetLowering::getAddrModeForFlags(unsigned Flags) const { 17439 // This is not a node we should be handling here. 17440 if (Flags == PPC::MOF_None) 17441 return PPC::AM_None; 17442 // Unaligned D-Forms are tried first, followed by the aligned D-Forms. 17443 for (auto FlagSet : AddrModesMap.at(PPC::AM_DForm)) 17444 if ((Flags & FlagSet) == FlagSet) 17445 return PPC::AM_DForm; 17446 for (auto FlagSet : AddrModesMap.at(PPC::AM_DSForm)) 17447 if ((Flags & FlagSet) == FlagSet) 17448 return PPC::AM_DSForm; 17449 for (auto FlagSet : AddrModesMap.at(PPC::AM_DQForm)) 17450 if ((Flags & FlagSet) == FlagSet) 17451 return PPC::AM_DQForm; 17452 for (auto FlagSet : AddrModesMap.at(PPC::AM_PrefixDForm)) 17453 if ((Flags & FlagSet) == FlagSet) 17454 return PPC::AM_PrefixDForm; 17455 // If no other forms are selected, return an X-Form as it is the most 17456 // general addressing mode. 17457 return PPC::AM_XForm; 17458 } 17459 17460 /// Set alignment flags based on whether or not the Frame Index is aligned. 17461 /// Utilized when computing flags for address computation when selecting 17462 /// load and store instructions. 17463 static void setAlignFlagsForFI(SDValue N, unsigned &FlagSet, 17464 SelectionDAG &DAG) { 17465 bool IsAdd = ((N.getOpcode() == ISD::ADD) || (N.getOpcode() == ISD::OR)); 17466 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(IsAdd ? N.getOperand(0) : N); 17467 if (!FI) 17468 return; 17469 const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); 17470 unsigned FrameIndexAlign = MFI.getObjectAlign(FI->getIndex()).value(); 17471 // If this is (add $FI, $S16Imm), the alignment flags are already set 17472 // based on the immediate. We just need to clear the alignment flags 17473 // if the FI alignment is weaker. 17474 if ((FrameIndexAlign % 4) != 0) 17475 FlagSet &= ~PPC::MOF_RPlusSImm16Mult4; 17476 if ((FrameIndexAlign % 16) != 0) 17477 FlagSet &= ~PPC::MOF_RPlusSImm16Mult16; 17478 // If the address is a plain FrameIndex, set alignment flags based on 17479 // FI alignment. 17480 if (!IsAdd) { 17481 if ((FrameIndexAlign % 4) == 0) 17482 FlagSet |= PPC::MOF_RPlusSImm16Mult4; 17483 if ((FrameIndexAlign % 16) == 0) 17484 FlagSet |= PPC::MOF_RPlusSImm16Mult16; 17485 } 17486 } 17487 17488 /// Given a node, compute flags that are used for address computation when 17489 /// selecting load and store instructions. The flags computed are stored in 17490 /// FlagSet. This function takes into account whether the node is a constant, 17491 /// an ADD, OR, or a constant, and computes the address flags accordingly. 17492 static void computeFlagsForAddressComputation(SDValue N, unsigned &FlagSet, 17493 SelectionDAG &DAG) { 17494 // Set the alignment flags for the node depending on if the node is 17495 // 4-byte or 16-byte aligned. 17496 auto SetAlignFlagsForImm = [&](uint64_t Imm) { 17497 if ((Imm & 0x3) == 0) 17498 FlagSet |= PPC::MOF_RPlusSImm16Mult4; 17499 if ((Imm & 0xf) == 0) 17500 FlagSet |= PPC::MOF_RPlusSImm16Mult16; 17501 }; 17502 17503 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) { 17504 // All 32-bit constants can be computed as LIS + Disp. 17505 const APInt &ConstImm = CN->getAPIntValue(); 17506 if (ConstImm.isSignedIntN(32)) { // Flag to handle 32-bit constants. 17507 FlagSet |= PPC::MOF_AddrIsSImm32; 17508 SetAlignFlagsForImm(ConstImm.getZExtValue()); 17509 setAlignFlagsForFI(N, FlagSet, DAG); 17510 } 17511 if (ConstImm.isSignedIntN(34)) // Flag to handle 34-bit constants. 17512 FlagSet |= PPC::MOF_RPlusSImm34; 17513 else // Let constant materialization handle large constants. 17514 FlagSet |= PPC::MOF_NotAddNorCst; 17515 } else if (N.getOpcode() == ISD::ADD || provablyDisjointOr(DAG, N)) { 17516 // This address can be represented as an addition of: 17517 // - Register + Imm16 (possibly a multiple of 4/16) 17518 // - Register + Imm34 17519 // - Register + PPCISD::Lo 17520 // - Register + Register 17521 // In any case, we won't have to match this as Base + Zero. 17522 SDValue RHS = N.getOperand(1); 17523 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(RHS)) { 17524 const APInt &ConstImm = CN->getAPIntValue(); 17525 if (ConstImm.isSignedIntN(16)) { 17526 FlagSet |= PPC::MOF_RPlusSImm16; // Signed 16-bit immediates. 17527 SetAlignFlagsForImm(ConstImm.getZExtValue()); 17528 setAlignFlagsForFI(N, FlagSet, DAG); 17529 } 17530 if (ConstImm.isSignedIntN(34)) 17531 FlagSet |= PPC::MOF_RPlusSImm34; // Signed 34-bit immediates. 17532 else 17533 FlagSet |= PPC::MOF_RPlusR; // Register. 17534 } else if (RHS.getOpcode() == PPCISD::Lo && 17535 !cast<ConstantSDNode>(RHS.getOperand(1))->getZExtValue()) 17536 FlagSet |= PPC::MOF_RPlusLo; // PPCISD::Lo. 17537 else 17538 FlagSet |= PPC::MOF_RPlusR; 17539 } else { // The address computation is not a constant or an addition. 17540 setAlignFlagsForFI(N, FlagSet, DAG); 17541 FlagSet |= PPC::MOF_NotAddNorCst; 17542 } 17543 } 17544 17545 static bool isPCRelNode(SDValue N) { 17546 return (N.getOpcode() == PPCISD::MAT_PCREL_ADDR || 17547 isValidPCRelNode<ConstantPoolSDNode>(N) || 17548 isValidPCRelNode<GlobalAddressSDNode>(N) || 17549 isValidPCRelNode<JumpTableSDNode>(N) || 17550 isValidPCRelNode<BlockAddressSDNode>(N)); 17551 } 17552 17553 /// computeMOFlags - Given a node N and it's Parent (a MemSDNode), compute 17554 /// the address flags of the load/store instruction that is to be matched. 17555 unsigned PPCTargetLowering::computeMOFlags(const SDNode *Parent, SDValue N, 17556 SelectionDAG &DAG) const { 17557 unsigned FlagSet = PPC::MOF_None; 17558 17559 // Compute subtarget flags. 17560 if (!Subtarget.hasP9Vector()) 17561 FlagSet |= PPC::MOF_SubtargetBeforeP9; 17562 else { 17563 FlagSet |= PPC::MOF_SubtargetP9; 17564 if (Subtarget.hasPrefixInstrs()) 17565 FlagSet |= PPC::MOF_SubtargetP10; 17566 } 17567 if (Subtarget.hasSPE()) 17568 FlagSet |= PPC::MOF_SubtargetSPE; 17569 17570 // Check if we have a PCRel node and return early. 17571 if ((FlagSet & PPC::MOF_SubtargetP10) && isPCRelNode(N)) 17572 return FlagSet; 17573 17574 // If the node is the paired load/store intrinsics, compute flags for 17575 // address computation and return early. 17576 unsigned ParentOp = Parent->getOpcode(); 17577 if (Subtarget.isISA3_1() && ((ParentOp == ISD::INTRINSIC_W_CHAIN) || 17578 (ParentOp == ISD::INTRINSIC_VOID))) { 17579 unsigned ID = cast<ConstantSDNode>(Parent->getOperand(1))->getZExtValue(); 17580 if ((ID == Intrinsic::ppc_vsx_lxvp) || (ID == Intrinsic::ppc_vsx_stxvp)) { 17581 SDValue IntrinOp = (ID == Intrinsic::ppc_vsx_lxvp) 17582 ? Parent->getOperand(2) 17583 : Parent->getOperand(3); 17584 computeFlagsForAddressComputation(IntrinOp, FlagSet, DAG); 17585 FlagSet |= PPC::MOF_Vector; 17586 return FlagSet; 17587 } 17588 } 17589 17590 // Mark this as something we don't want to handle here if it is atomic 17591 // or pre-increment instruction. 17592 if (const LSBaseSDNode *LSB = dyn_cast<LSBaseSDNode>(Parent)) 17593 if (LSB->isIndexed()) 17594 return PPC::MOF_None; 17595 17596 // Compute in-memory type flags. This is based on if there are scalars, 17597 // floats or vectors. 17598 const MemSDNode *MN = dyn_cast<MemSDNode>(Parent); 17599 assert(MN && "Parent should be a MemSDNode!"); 17600 EVT MemVT = MN->getMemoryVT(); 17601 unsigned Size = MemVT.getSizeInBits(); 17602 if (MemVT.isScalarInteger()) { 17603 assert(Size <= 128 && 17604 "Not expecting scalar integers larger than 16 bytes!"); 17605 if (Size < 32) 17606 FlagSet |= PPC::MOF_SubWordInt; 17607 else if (Size == 32) 17608 FlagSet |= PPC::MOF_WordInt; 17609 else 17610 FlagSet |= PPC::MOF_DoubleWordInt; 17611 } else if (MemVT.isVector() && !MemVT.isFloatingPoint()) { // Integer vectors. 17612 if (Size == 128) 17613 FlagSet |= PPC::MOF_Vector; 17614 else if (Size == 256) { 17615 assert(Subtarget.pairedVectorMemops() && 17616 "256-bit vectors are only available when paired vector memops is " 17617 "enabled!"); 17618 FlagSet |= PPC::MOF_Vector; 17619 } else 17620 llvm_unreachable("Not expecting illegal vectors!"); 17621 } else { // Floating point type: can be scalar, f128 or vector types. 17622 if (Size == 32 || Size == 64) 17623 FlagSet |= PPC::MOF_ScalarFloat; 17624 else if (MemVT == MVT::f128 || MemVT.isVector()) 17625 FlagSet |= PPC::MOF_Vector; 17626 else 17627 llvm_unreachable("Not expecting illegal scalar floats!"); 17628 } 17629 17630 // Compute flags for address computation. 17631 computeFlagsForAddressComputation(N, FlagSet, DAG); 17632 17633 // Compute type extension flags. 17634 if (const LoadSDNode *LN = dyn_cast<LoadSDNode>(Parent)) { 17635 switch (LN->getExtensionType()) { 17636 case ISD::SEXTLOAD: 17637 FlagSet |= PPC::MOF_SExt; 17638 break; 17639 case ISD::EXTLOAD: 17640 case ISD::ZEXTLOAD: 17641 FlagSet |= PPC::MOF_ZExt; 17642 break; 17643 case ISD::NON_EXTLOAD: 17644 FlagSet |= PPC::MOF_NoExt; 17645 break; 17646 } 17647 } else 17648 FlagSet |= PPC::MOF_NoExt; 17649 17650 // For integers, no extension is the same as zero extension. 17651 // We set the extension mode to zero extension so we don't have 17652 // to add separate entries in AddrModesMap for loads and stores. 17653 if (MemVT.isScalarInteger() && (FlagSet & PPC::MOF_NoExt)) { 17654 FlagSet |= PPC::MOF_ZExt; 17655 FlagSet &= ~PPC::MOF_NoExt; 17656 } 17657 17658 // If we don't have prefixed instructions, 34-bit constants should be 17659 // treated as PPC::MOF_NotAddNorCst so they can match D-Forms. 17660 bool IsNonP1034BitConst = 17661 ((PPC::MOF_RPlusSImm34 | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubtargetP10) & 17662 FlagSet) == PPC::MOF_RPlusSImm34; 17663 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::OR && 17664 IsNonP1034BitConst) 17665 FlagSet |= PPC::MOF_NotAddNorCst; 17666 17667 return FlagSet; 17668 } 17669 17670 /// SelectForceXFormMode - Given the specified address, force it to be 17671 /// represented as an indexed [r+r] operation (an XForm instruction). 17672 PPC::AddrMode PPCTargetLowering::SelectForceXFormMode(SDValue N, SDValue &Disp, 17673 SDValue &Base, 17674 SelectionDAG &DAG) const { 17675 17676 PPC::AddrMode Mode = PPC::AM_XForm; 17677 int16_t ForceXFormImm = 0; 17678 if (provablyDisjointOr(DAG, N) && 17679 !isIntS16Immediate(N.getOperand(1), ForceXFormImm)) { 17680 Disp = N.getOperand(0); 17681 Base = N.getOperand(1); 17682 return Mode; 17683 } 17684 17685 // If the address is the result of an add, we will utilize the fact that the 17686 // address calculation includes an implicit add. However, we can reduce 17687 // register pressure if we do not materialize a constant just for use as the 17688 // index register. We only get rid of the add if it is not an add of a 17689 // value and a 16-bit signed constant and both have a single use. 17690 if (N.getOpcode() == ISD::ADD && 17691 (!isIntS16Immediate(N.getOperand(1), ForceXFormImm) || 17692 !N.getOperand(1).hasOneUse() || !N.getOperand(0).hasOneUse())) { 17693 Disp = N.getOperand(0); 17694 Base = N.getOperand(1); 17695 return Mode; 17696 } 17697 17698 // Otherwise, use R0 as the base register. 17699 Disp = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO, 17700 N.getValueType()); 17701 Base = N; 17702 17703 return Mode; 17704 } 17705 17706 bool PPCTargetLowering::splitValueIntoRegisterParts( 17707 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, 17708 unsigned NumParts, MVT PartVT, Optional<CallingConv::ID> CC) const { 17709 EVT ValVT = Val.getValueType(); 17710 // If we are splitting a scalar integer into f64 parts (i.e. so they 17711 // can be placed into VFRC registers), we need to zero extend and 17712 // bitcast the values. This will ensure the value is placed into a 17713 // VSR using direct moves or stack operations as needed. 17714 if (PartVT == MVT::f64 && 17715 (ValVT == MVT::i32 || ValVT == MVT::i16 || ValVT == MVT::i8)) { 17716 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Val); 17717 Val = DAG.getNode(ISD::BITCAST, DL, MVT::f64, Val); 17718 Parts[0] = Val; 17719 return true; 17720 } 17721 return false; 17722 } 17723 17724 // If we happen to match to an aligned D-Form, check if the Frame Index is 17725 // adequately aligned. If it is not, reset the mode to match to X-Form. 17726 static void setXFormForUnalignedFI(SDValue N, unsigned Flags, 17727 PPC::AddrMode &Mode) { 17728 if (!isa<FrameIndexSDNode>(N)) 17729 return; 17730 if ((Mode == PPC::AM_DSForm && !(Flags & PPC::MOF_RPlusSImm16Mult4)) || 17731 (Mode == PPC::AM_DQForm && !(Flags & PPC::MOF_RPlusSImm16Mult16))) 17732 Mode = PPC::AM_XForm; 17733 } 17734 17735 /// SelectOptimalAddrMode - Based on a node N and it's Parent (a MemSDNode), 17736 /// compute the address flags of the node, get the optimal address mode based 17737 /// on the flags, and set the Base and Disp based on the address mode. 17738 PPC::AddrMode PPCTargetLowering::SelectOptimalAddrMode(const SDNode *Parent, 17739 SDValue N, SDValue &Disp, 17740 SDValue &Base, 17741 SelectionDAG &DAG, 17742 MaybeAlign Align) const { 17743 SDLoc DL(Parent); 17744 17745 // Compute the address flags. 17746 unsigned Flags = computeMOFlags(Parent, N, DAG); 17747 17748 // Get the optimal address mode based on the Flags. 17749 PPC::AddrMode Mode = getAddrModeForFlags(Flags); 17750 17751 // If the address mode is DS-Form or DQ-Form, check if the FI is aligned. 17752 // Select an X-Form load if it is not. 17753 setXFormForUnalignedFI(N, Flags, Mode); 17754 17755 // Set the mode to PC-Relative addressing mode if we have a valid PC-Rel node. 17756 if ((Mode == PPC::AM_XForm) && isPCRelNode(N)) { 17757 assert(Subtarget.isUsingPCRelativeCalls() && 17758 "Must be using PC-Relative calls when a valid PC-Relative node is " 17759 "present!"); 17760 Mode = PPC::AM_PCRel; 17761 } 17762 17763 // Set Base and Disp accordingly depending on the address mode. 17764 switch (Mode) { 17765 case PPC::AM_DForm: 17766 case PPC::AM_DSForm: 17767 case PPC::AM_DQForm: { 17768 // This is a register plus a 16-bit immediate. The base will be the 17769 // register and the displacement will be the immediate unless it 17770 // isn't sufficiently aligned. 17771 if (Flags & PPC::MOF_RPlusSImm16) { 17772 SDValue Op0 = N.getOperand(0); 17773 SDValue Op1 = N.getOperand(1); 17774 int16_t Imm = cast<ConstantSDNode>(Op1)->getAPIntValue().getZExtValue(); 17775 if (!Align || isAligned(*Align, Imm)) { 17776 Disp = DAG.getTargetConstant(Imm, DL, N.getValueType()); 17777 Base = Op0; 17778 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op0)) { 17779 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); 17780 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); 17781 } 17782 break; 17783 } 17784 } 17785 // This is a register plus the @lo relocation. The base is the register 17786 // and the displacement is the global address. 17787 else if (Flags & PPC::MOF_RPlusLo) { 17788 Disp = N.getOperand(1).getOperand(0); // The global address. 17789 assert(Disp.getOpcode() == ISD::TargetGlobalAddress || 17790 Disp.getOpcode() == ISD::TargetGlobalTLSAddress || 17791 Disp.getOpcode() == ISD::TargetConstantPool || 17792 Disp.getOpcode() == ISD::TargetJumpTable); 17793 Base = N.getOperand(0); 17794 break; 17795 } 17796 // This is a constant address at most 32 bits. The base will be 17797 // zero or load-immediate-shifted and the displacement will be 17798 // the low 16 bits of the address. 17799 else if (Flags & PPC::MOF_AddrIsSImm32) { 17800 auto *CN = cast<ConstantSDNode>(N); 17801 EVT CNType = CN->getValueType(0); 17802 uint64_t CNImm = CN->getZExtValue(); 17803 // If this address fits entirely in a 16-bit sext immediate field, codegen 17804 // this as "d, 0". 17805 int16_t Imm; 17806 if (isIntS16Immediate(CN, Imm) && (!Align || isAligned(*Align, Imm))) { 17807 Disp = DAG.getTargetConstant(Imm, DL, CNType); 17808 Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO, 17809 CNType); 17810 break; 17811 } 17812 // Handle 32-bit sext immediate with LIS + Addr mode. 17813 if ((CNType == MVT::i32 || isInt<32>(CNImm)) && 17814 (!Align || isAligned(*Align, CNImm))) { 17815 int32_t Addr = (int32_t)CNImm; 17816 // Otherwise, break this down into LIS + Disp. 17817 Disp = DAG.getTargetConstant((int16_t)Addr, DL, MVT::i32); 17818 Base = 17819 DAG.getTargetConstant((Addr - (int16_t)Addr) >> 16, DL, MVT::i32); 17820 uint32_t LIS = CNType == MVT::i32 ? PPC::LIS : PPC::LIS8; 17821 Base = SDValue(DAG.getMachineNode(LIS, DL, CNType, Base), 0); 17822 break; 17823 } 17824 } 17825 // Otherwise, the PPC:MOF_NotAdd flag is set. Load/Store is Non-foldable. 17826 Disp = DAG.getTargetConstant(0, DL, getPointerTy(DAG.getDataLayout())); 17827 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N)) { 17828 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); 17829 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); 17830 } else 17831 Base = N; 17832 break; 17833 } 17834 case PPC::AM_PrefixDForm: { 17835 int64_t Imm34 = 0; 17836 unsigned Opcode = N.getOpcode(); 17837 if (((Opcode == ISD::ADD) || (Opcode == ISD::OR)) && 17838 (isIntS34Immediate(N.getOperand(1), Imm34))) { 17839 // N is an Add/OR Node, and it's operand is a 34-bit signed immediate. 17840 Disp = DAG.getTargetConstant(Imm34, DL, N.getValueType()); 17841 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N.getOperand(0))) 17842 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); 17843 else 17844 Base = N.getOperand(0); 17845 } else if (isIntS34Immediate(N, Imm34)) { 17846 // The address is a 34-bit signed immediate. 17847 Disp = DAG.getTargetConstant(Imm34, DL, N.getValueType()); 17848 Base = DAG.getRegister(PPC::ZERO8, N.getValueType()); 17849 } 17850 break; 17851 } 17852 case PPC::AM_PCRel: { 17853 // When selecting PC-Relative instructions, "Base" is not utilized as 17854 // we select the address as [PC+imm]. 17855 Disp = N; 17856 break; 17857 } 17858 case PPC::AM_None: 17859 break; 17860 default: { // By default, X-Form is always available to be selected. 17861 // When a frame index is not aligned, we also match by XForm. 17862 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N); 17863 Base = FI ? N : N.getOperand(1); 17864 Disp = FI ? DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO, 17865 N.getValueType()) 17866 : N.getOperand(0); 17867 break; 17868 } 17869 } 17870 return Mode; 17871 } 17872 17873 CCAssignFn *PPCTargetLowering::ccAssignFnForCall(CallingConv::ID CC, 17874 bool Return, 17875 bool IsVarArg) const { 17876 switch (CC) { 17877 case CallingConv::Cold: 17878 return (Return ? RetCC_PPC_Cold : CC_PPC64_ELF_FIS); 17879 default: 17880 return CC_PPC64_ELF_FIS; 17881 } 17882 } 17883 17884 TargetLowering::AtomicExpansionKind 17885 PPCTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const { 17886 unsigned Size = AI->getType()->getPrimitiveSizeInBits(); 17887 if (EnableQuadwordAtomics && Subtarget.hasQuadwordAtomics() && Size == 128) 17888 return AtomicExpansionKind::MaskedIntrinsic; 17889 return TargetLowering::shouldExpandAtomicRMWInIR(AI); 17890 } 17891 17892 TargetLowering::AtomicExpansionKind 17893 PPCTargetLowering::shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const { 17894 unsigned Size = AI->getNewValOperand()->getType()->getPrimitiveSizeInBits(); 17895 if (EnableQuadwordAtomics && Subtarget.hasQuadwordAtomics() && Size == 128) 17896 return AtomicExpansionKind::MaskedIntrinsic; 17897 return TargetLowering::shouldExpandAtomicCmpXchgInIR(AI); 17898 } 17899 17900 static Intrinsic::ID 17901 getIntrinsicForAtomicRMWBinOp128(AtomicRMWInst::BinOp BinOp) { 17902 switch (BinOp) { 17903 default: 17904 llvm_unreachable("Unexpected AtomicRMW BinOp"); 17905 case AtomicRMWInst::Xchg: 17906 return Intrinsic::ppc_atomicrmw_xchg_i128; 17907 case AtomicRMWInst::Add: 17908 return Intrinsic::ppc_atomicrmw_add_i128; 17909 case AtomicRMWInst::Sub: 17910 return Intrinsic::ppc_atomicrmw_sub_i128; 17911 case AtomicRMWInst::And: 17912 return Intrinsic::ppc_atomicrmw_and_i128; 17913 case AtomicRMWInst::Or: 17914 return Intrinsic::ppc_atomicrmw_or_i128; 17915 case AtomicRMWInst::Xor: 17916 return Intrinsic::ppc_atomicrmw_xor_i128; 17917 case AtomicRMWInst::Nand: 17918 return Intrinsic::ppc_atomicrmw_nand_i128; 17919 } 17920 } 17921 17922 Value *PPCTargetLowering::emitMaskedAtomicRMWIntrinsic( 17923 IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, 17924 Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const { 17925 assert(EnableQuadwordAtomics && Subtarget.hasQuadwordAtomics() && 17926 "Only support quadword now"); 17927 Module *M = Builder.GetInsertBlock()->getParent()->getParent(); 17928 Type *ValTy = AlignedAddr->getType()->getPointerElementType(); 17929 assert(ValTy->getPrimitiveSizeInBits() == 128); 17930 Function *RMW = Intrinsic::getDeclaration( 17931 M, getIntrinsicForAtomicRMWBinOp128(AI->getOperation())); 17932 Type *Int64Ty = Type::getInt64Ty(M->getContext()); 17933 Value *IncrLo = Builder.CreateTrunc(Incr, Int64Ty, "incr_lo"); 17934 Value *IncrHi = 17935 Builder.CreateTrunc(Builder.CreateLShr(Incr, 64), Int64Ty, "incr_hi"); 17936 Value *Addr = 17937 Builder.CreateBitCast(AlignedAddr, Type::getInt8PtrTy(M->getContext())); 17938 Value *LoHi = Builder.CreateCall(RMW, {Addr, IncrLo, IncrHi}); 17939 Value *Lo = Builder.CreateExtractValue(LoHi, 0, "lo"); 17940 Value *Hi = Builder.CreateExtractValue(LoHi, 1, "hi"); 17941 Lo = Builder.CreateZExt(Lo, ValTy, "lo64"); 17942 Hi = Builder.CreateZExt(Hi, ValTy, "hi64"); 17943 return Builder.CreateOr( 17944 Lo, Builder.CreateShl(Hi, ConstantInt::get(ValTy, 64)), "val64"); 17945 } 17946 17947 Value *PPCTargetLowering::emitMaskedAtomicCmpXchgIntrinsic( 17948 IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, 17949 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const { 17950 assert(EnableQuadwordAtomics && Subtarget.hasQuadwordAtomics() && 17951 "Only support quadword now"); 17952 Module *M = Builder.GetInsertBlock()->getParent()->getParent(); 17953 Type *ValTy = AlignedAddr->getType()->getPointerElementType(); 17954 assert(ValTy->getPrimitiveSizeInBits() == 128); 17955 Function *IntCmpXchg = 17956 Intrinsic::getDeclaration(M, Intrinsic::ppc_cmpxchg_i128); 17957 Type *Int64Ty = Type::getInt64Ty(M->getContext()); 17958 Value *CmpLo = Builder.CreateTrunc(CmpVal, Int64Ty, "cmp_lo"); 17959 Value *CmpHi = 17960 Builder.CreateTrunc(Builder.CreateLShr(CmpVal, 64), Int64Ty, "cmp_hi"); 17961 Value *NewLo = Builder.CreateTrunc(NewVal, Int64Ty, "new_lo"); 17962 Value *NewHi = 17963 Builder.CreateTrunc(Builder.CreateLShr(NewVal, 64), Int64Ty, "new_hi"); 17964 Value *Addr = 17965 Builder.CreateBitCast(AlignedAddr, Type::getInt8PtrTy(M->getContext())); 17966 emitLeadingFence(Builder, CI, Ord); 17967 Value *LoHi = 17968 Builder.CreateCall(IntCmpXchg, {Addr, CmpLo, CmpHi, NewLo, NewHi}); 17969 emitTrailingFence(Builder, CI, Ord); 17970 Value *Lo = Builder.CreateExtractValue(LoHi, 0, "lo"); 17971 Value *Hi = Builder.CreateExtractValue(LoHi, 1, "hi"); 17972 Lo = Builder.CreateZExt(Lo, ValTy, "lo64"); 17973 Hi = Builder.CreateZExt(Hi, ValTy, "hi64"); 17974 return Builder.CreateOr( 17975 Lo, Builder.CreateShl(Hi, ConstantInt::get(ValTy, 64)), "val64"); 17976 } 17977