1 //===-- PPCISelLowering.cpp - PPC DAG Lowering Implementation -------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the PPCISelLowering class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "PPCISelLowering.h" 14 #include "MCTargetDesc/PPCPredicates.h" 15 #include "PPC.h" 16 #include "PPCCCState.h" 17 #include "PPCCallingConv.h" 18 #include "PPCFrameLowering.h" 19 #include "PPCInstrInfo.h" 20 #include "PPCMachineFunctionInfo.h" 21 #include "PPCPerfectShuffle.h" 22 #include "PPCRegisterInfo.h" 23 #include "PPCSubtarget.h" 24 #include "PPCTargetMachine.h" 25 #include "llvm/ADT/APFloat.h" 26 #include "llvm/ADT/APInt.h" 27 #include "llvm/ADT/ArrayRef.h" 28 #include "llvm/ADT/DenseMap.h" 29 #include "llvm/ADT/None.h" 30 #include "llvm/ADT/STLExtras.h" 31 #include "llvm/ADT/SmallPtrSet.h" 32 #include "llvm/ADT/SmallSet.h" 33 #include "llvm/ADT/SmallVector.h" 34 #include "llvm/ADT/Statistic.h" 35 #include "llvm/ADT/StringRef.h" 36 #include "llvm/ADT/StringSwitch.h" 37 #include "llvm/CodeGen/CallingConvLower.h" 38 #include "llvm/CodeGen/ISDOpcodes.h" 39 #include "llvm/CodeGen/MachineBasicBlock.h" 40 #include "llvm/CodeGen/MachineFrameInfo.h" 41 #include "llvm/CodeGen/MachineFunction.h" 42 #include "llvm/CodeGen/MachineInstr.h" 43 #include "llvm/CodeGen/MachineInstrBuilder.h" 44 #include "llvm/CodeGen/MachineJumpTableInfo.h" 45 #include "llvm/CodeGen/MachineLoopInfo.h" 46 #include "llvm/CodeGen/MachineMemOperand.h" 47 #include "llvm/CodeGen/MachineModuleInfo.h" 48 #include "llvm/CodeGen/MachineOperand.h" 49 #include "llvm/CodeGen/MachineRegisterInfo.h" 50 #include "llvm/CodeGen/RuntimeLibcalls.h" 51 #include "llvm/CodeGen/SelectionDAG.h" 52 #include "llvm/CodeGen/SelectionDAGNodes.h" 53 #include "llvm/CodeGen/TargetInstrInfo.h" 54 #include "llvm/CodeGen/TargetLowering.h" 55 #include "llvm/CodeGen/TargetRegisterInfo.h" 56 #include "llvm/CodeGen/ValueTypes.h" 57 #include "llvm/IR/CallSite.h" 58 #include "llvm/IR/CallingConv.h" 59 #include "llvm/IR/Constant.h" 60 #include "llvm/IR/Constants.h" 61 #include "llvm/IR/DataLayout.h" 62 #include "llvm/IR/DebugLoc.h" 63 #include "llvm/IR/DerivedTypes.h" 64 #include "llvm/IR/Function.h" 65 #include "llvm/IR/GlobalValue.h" 66 #include "llvm/IR/IRBuilder.h" 67 #include "llvm/IR/Instructions.h" 68 #include "llvm/IR/Intrinsics.h" 69 #include "llvm/IR/Module.h" 70 #include "llvm/IR/Type.h" 71 #include "llvm/IR/Use.h" 72 #include "llvm/IR/Value.h" 73 #include "llvm/MC/MCContext.h" 74 #include "llvm/MC/MCExpr.h" 75 #include "llvm/MC/MCRegisterInfo.h" 76 #include "llvm/MC/MCSymbolXCOFF.h" 77 #include "llvm/Support/AtomicOrdering.h" 78 #include "llvm/Support/BranchProbability.h" 79 #include "llvm/Support/Casting.h" 80 #include "llvm/Support/CodeGen.h" 81 #include "llvm/Support/CommandLine.h" 82 #include "llvm/Support/Compiler.h" 83 #include "llvm/Support/Debug.h" 84 #include "llvm/Support/ErrorHandling.h" 85 #include "llvm/Support/Format.h" 86 #include "llvm/Support/KnownBits.h" 87 #include "llvm/Support/MachineValueType.h" 88 #include "llvm/Support/MathExtras.h" 89 #include "llvm/Support/raw_ostream.h" 90 #include "llvm/Target/TargetMachine.h" 91 #include "llvm/Target/TargetOptions.h" 92 #include <algorithm> 93 #include <cassert> 94 #include <cstdint> 95 #include <iterator> 96 #include <list> 97 #include <utility> 98 #include <vector> 99 100 using namespace llvm; 101 102 #define DEBUG_TYPE "ppc-lowering" 103 104 static cl::opt<bool> DisablePPCPreinc("disable-ppc-preinc", 105 cl::desc("disable preincrement load/store generation on PPC"), cl::Hidden); 106 107 static cl::opt<bool> DisableILPPref("disable-ppc-ilp-pref", 108 cl::desc("disable setting the node scheduling preference to ILP on PPC"), cl::Hidden); 109 110 static cl::opt<bool> DisablePPCUnaligned("disable-ppc-unaligned", 111 cl::desc("disable unaligned load/store generation on PPC"), cl::Hidden); 112 113 static cl::opt<bool> DisableSCO("disable-ppc-sco", 114 cl::desc("disable sibling call optimization on ppc"), cl::Hidden); 115 116 static cl::opt<bool> DisableInnermostLoopAlign32("disable-ppc-innermost-loop-align32", 117 cl::desc("don't always align innermost loop to 32 bytes on ppc"), cl::Hidden); 118 119 static cl::opt<bool> EnableQuadPrecision("enable-ppc-quad-precision", 120 cl::desc("enable quad precision float support on ppc"), cl::Hidden); 121 122 STATISTIC(NumTailCalls, "Number of tail calls"); 123 STATISTIC(NumSiblingCalls, "Number of sibling calls"); 124 125 static bool isNByteElemShuffleMask(ShuffleVectorSDNode *, unsigned, int); 126 127 static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl); 128 129 // FIXME: Remove this once the bug has been fixed! 130 extern cl::opt<bool> ANDIGlueBug; 131 132 PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, 133 const PPCSubtarget &STI) 134 : TargetLowering(TM), Subtarget(STI) { 135 // Use _setjmp/_longjmp instead of setjmp/longjmp. 136 setUseUnderscoreSetJmp(true); 137 setUseUnderscoreLongJmp(true); 138 139 // On PPC32/64, arguments smaller than 4/8 bytes are extended, so all 140 // arguments are at least 4/8 bytes aligned. 141 bool isPPC64 = Subtarget.isPPC64(); 142 setMinStackArgumentAlignment(isPPC64 ? 8:4); 143 144 // Set up the register classes. 145 addRegisterClass(MVT::i32, &PPC::GPRCRegClass); 146 if (!useSoftFloat()) { 147 if (hasSPE()) { 148 addRegisterClass(MVT::f32, &PPC::SPE4RCRegClass); 149 addRegisterClass(MVT::f64, &PPC::SPERCRegClass); 150 } else { 151 addRegisterClass(MVT::f32, &PPC::F4RCRegClass); 152 addRegisterClass(MVT::f64, &PPC::F8RCRegClass); 153 } 154 } 155 156 // Match BITREVERSE to customized fast code sequence in the td file. 157 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); 158 setOperationAction(ISD::BITREVERSE, MVT::i64, Legal); 159 160 // Sub-word ATOMIC_CMP_SWAP need to ensure that the input is zero-extended. 161 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); 162 163 // PowerPC has an i16 but no i8 (or i1) SEXTLOAD. 164 for (MVT VT : MVT::integer_valuetypes()) { 165 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); 166 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Expand); 167 } 168 169 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 170 171 // PowerPC has pre-inc load and store's. 172 setIndexedLoadAction(ISD::PRE_INC, MVT::i1, Legal); 173 setIndexedLoadAction(ISD::PRE_INC, MVT::i8, Legal); 174 setIndexedLoadAction(ISD::PRE_INC, MVT::i16, Legal); 175 setIndexedLoadAction(ISD::PRE_INC, MVT::i32, Legal); 176 setIndexedLoadAction(ISD::PRE_INC, MVT::i64, Legal); 177 setIndexedStoreAction(ISD::PRE_INC, MVT::i1, Legal); 178 setIndexedStoreAction(ISD::PRE_INC, MVT::i8, Legal); 179 setIndexedStoreAction(ISD::PRE_INC, MVT::i16, Legal); 180 setIndexedStoreAction(ISD::PRE_INC, MVT::i32, Legal); 181 setIndexedStoreAction(ISD::PRE_INC, MVT::i64, Legal); 182 if (!Subtarget.hasSPE()) { 183 setIndexedLoadAction(ISD::PRE_INC, MVT::f32, Legal); 184 setIndexedLoadAction(ISD::PRE_INC, MVT::f64, Legal); 185 setIndexedStoreAction(ISD::PRE_INC, MVT::f32, Legal); 186 setIndexedStoreAction(ISD::PRE_INC, MVT::f64, Legal); 187 } 188 189 // PowerPC uses ADDC/ADDE/SUBC/SUBE to propagate carry. 190 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 }; 191 for (MVT VT : ScalarIntVTs) { 192 setOperationAction(ISD::ADDC, VT, Legal); 193 setOperationAction(ISD::ADDE, VT, Legal); 194 setOperationAction(ISD::SUBC, VT, Legal); 195 setOperationAction(ISD::SUBE, VT, Legal); 196 } 197 198 if (Subtarget.useCRBits()) { 199 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 200 201 if (isPPC64 || Subtarget.hasFPCVT()) { 202 setOperationAction(ISD::SINT_TO_FP, MVT::i1, Promote); 203 AddPromotedToType (ISD::SINT_TO_FP, MVT::i1, 204 isPPC64 ? MVT::i64 : MVT::i32); 205 setOperationAction(ISD::UINT_TO_FP, MVT::i1, Promote); 206 AddPromotedToType(ISD::UINT_TO_FP, MVT::i1, 207 isPPC64 ? MVT::i64 : MVT::i32); 208 } else { 209 setOperationAction(ISD::SINT_TO_FP, MVT::i1, Custom); 210 setOperationAction(ISD::UINT_TO_FP, MVT::i1, Custom); 211 } 212 213 // PowerPC does not support direct load/store of condition registers. 214 setOperationAction(ISD::LOAD, MVT::i1, Custom); 215 setOperationAction(ISD::STORE, MVT::i1, Custom); 216 217 // FIXME: Remove this once the ANDI glue bug is fixed: 218 if (ANDIGlueBug) 219 setOperationAction(ISD::TRUNCATE, MVT::i1, Custom); 220 221 for (MVT VT : MVT::integer_valuetypes()) { 222 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); 223 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote); 224 setTruncStoreAction(VT, MVT::i1, Expand); 225 } 226 227 addRegisterClass(MVT::i1, &PPC::CRBITRCRegClass); 228 } 229 230 // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on 231 // PPC (the libcall is not available). 232 setOperationAction(ISD::FP_TO_SINT, MVT::ppcf128, Custom); 233 setOperationAction(ISD::FP_TO_UINT, MVT::ppcf128, Custom); 234 235 // We do not currently implement these libm ops for PowerPC. 236 setOperationAction(ISD::FFLOOR, MVT::ppcf128, Expand); 237 setOperationAction(ISD::FCEIL, MVT::ppcf128, Expand); 238 setOperationAction(ISD::FTRUNC, MVT::ppcf128, Expand); 239 setOperationAction(ISD::FRINT, MVT::ppcf128, Expand); 240 setOperationAction(ISD::FNEARBYINT, MVT::ppcf128, Expand); 241 setOperationAction(ISD::FREM, MVT::ppcf128, Expand); 242 243 // PowerPC has no SREM/UREM instructions unless we are on P9 244 // On P9 we may use a hardware instruction to compute the remainder. 245 // The instructions are not legalized directly because in the cases where the 246 // result of both the remainder and the division is required it is more 247 // efficient to compute the remainder from the result of the division rather 248 // than use the remainder instruction. 249 if (Subtarget.isISA3_0()) { 250 setOperationAction(ISD::SREM, MVT::i32, Custom); 251 setOperationAction(ISD::UREM, MVT::i32, Custom); 252 setOperationAction(ISD::SREM, MVT::i64, Custom); 253 setOperationAction(ISD::UREM, MVT::i64, Custom); 254 } else { 255 setOperationAction(ISD::SREM, MVT::i32, Expand); 256 setOperationAction(ISD::UREM, MVT::i32, Expand); 257 setOperationAction(ISD::SREM, MVT::i64, Expand); 258 setOperationAction(ISD::UREM, MVT::i64, Expand); 259 } 260 261 // Don't use SMUL_LOHI/UMUL_LOHI or SDIVREM/UDIVREM to lower SREM/UREM. 262 setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand); 263 setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand); 264 setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand); 265 setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand); 266 setOperationAction(ISD::UDIVREM, MVT::i32, Expand); 267 setOperationAction(ISD::SDIVREM, MVT::i32, Expand); 268 setOperationAction(ISD::UDIVREM, MVT::i64, Expand); 269 setOperationAction(ISD::SDIVREM, MVT::i64, Expand); 270 271 // We don't support sin/cos/sqrt/fmod/pow 272 setOperationAction(ISD::FSIN , MVT::f64, Expand); 273 setOperationAction(ISD::FCOS , MVT::f64, Expand); 274 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); 275 setOperationAction(ISD::FREM , MVT::f64, Expand); 276 setOperationAction(ISD::FPOW , MVT::f64, Expand); 277 setOperationAction(ISD::FSIN , MVT::f32, Expand); 278 setOperationAction(ISD::FCOS , MVT::f32, Expand); 279 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); 280 setOperationAction(ISD::FREM , MVT::f32, Expand); 281 setOperationAction(ISD::FPOW , MVT::f32, Expand); 282 if (Subtarget.hasSPE()) { 283 setOperationAction(ISD::FMA , MVT::f64, Expand); 284 setOperationAction(ISD::FMA , MVT::f32, Expand); 285 } else { 286 setOperationAction(ISD::FMA , MVT::f64, Legal); 287 setOperationAction(ISD::FMA , MVT::f32, Legal); 288 } 289 290 setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom); 291 292 // If we're enabling GP optimizations, use hardware square root 293 if (!Subtarget.hasFSQRT() && 294 !(TM.Options.UnsafeFPMath && Subtarget.hasFRSQRTE() && 295 Subtarget.hasFRE())) 296 setOperationAction(ISD::FSQRT, MVT::f64, Expand); 297 298 if (!Subtarget.hasFSQRT() && 299 !(TM.Options.UnsafeFPMath && Subtarget.hasFRSQRTES() && 300 Subtarget.hasFRES())) 301 setOperationAction(ISD::FSQRT, MVT::f32, Expand); 302 303 if (Subtarget.hasFCPSGN()) { 304 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Legal); 305 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Legal); 306 } else { 307 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 308 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); 309 } 310 311 if (Subtarget.hasFPRND()) { 312 setOperationAction(ISD::FFLOOR, MVT::f64, Legal); 313 setOperationAction(ISD::FCEIL, MVT::f64, Legal); 314 setOperationAction(ISD::FTRUNC, MVT::f64, Legal); 315 setOperationAction(ISD::FROUND, MVT::f64, Legal); 316 317 setOperationAction(ISD::FFLOOR, MVT::f32, Legal); 318 setOperationAction(ISD::FCEIL, MVT::f32, Legal); 319 setOperationAction(ISD::FTRUNC, MVT::f32, Legal); 320 setOperationAction(ISD::FROUND, MVT::f32, Legal); 321 } 322 323 // PowerPC does not have BSWAP, but we can use vector BSWAP instruction xxbrd 324 // to speed up scalar BSWAP64. 325 // CTPOP or CTTZ were introduced in P8/P9 respectively 326 setOperationAction(ISD::BSWAP, MVT::i32 , Expand); 327 if (Subtarget.hasP9Vector()) 328 setOperationAction(ISD::BSWAP, MVT::i64 , Custom); 329 else 330 setOperationAction(ISD::BSWAP, MVT::i64 , Expand); 331 if (Subtarget.isISA3_0()) { 332 setOperationAction(ISD::CTTZ , MVT::i32 , Legal); 333 setOperationAction(ISD::CTTZ , MVT::i64 , Legal); 334 } else { 335 setOperationAction(ISD::CTTZ , MVT::i32 , Expand); 336 setOperationAction(ISD::CTTZ , MVT::i64 , Expand); 337 } 338 339 if (Subtarget.hasPOPCNTD() == PPCSubtarget::POPCNTD_Fast) { 340 setOperationAction(ISD::CTPOP, MVT::i32 , Legal); 341 setOperationAction(ISD::CTPOP, MVT::i64 , Legal); 342 } else { 343 setOperationAction(ISD::CTPOP, MVT::i32 , Expand); 344 setOperationAction(ISD::CTPOP, MVT::i64 , Expand); 345 } 346 347 // PowerPC does not have ROTR 348 setOperationAction(ISD::ROTR, MVT::i32 , Expand); 349 setOperationAction(ISD::ROTR, MVT::i64 , Expand); 350 351 if (!Subtarget.useCRBits()) { 352 // PowerPC does not have Select 353 setOperationAction(ISD::SELECT, MVT::i32, Expand); 354 setOperationAction(ISD::SELECT, MVT::i64, Expand); 355 setOperationAction(ISD::SELECT, MVT::f32, Expand); 356 setOperationAction(ISD::SELECT, MVT::f64, Expand); 357 } 358 359 // PowerPC wants to turn select_cc of FP into fsel when possible. 360 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); 361 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom); 362 363 // PowerPC wants to optimize integer setcc a bit 364 if (!Subtarget.useCRBits()) 365 setOperationAction(ISD::SETCC, MVT::i32, Custom); 366 367 // PowerPC does not have BRCOND which requires SetCC 368 if (!Subtarget.useCRBits()) 369 setOperationAction(ISD::BRCOND, MVT::Other, Expand); 370 371 setOperationAction(ISD::BR_JT, MVT::Other, Expand); 372 373 if (Subtarget.hasSPE()) { 374 // SPE has built-in conversions 375 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Legal); 376 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Legal); 377 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Legal); 378 } else { 379 // PowerPC turns FP_TO_SINT into FCTIWZ and some load/stores. 380 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); 381 382 // PowerPC does not have [U|S]INT_TO_FP 383 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Expand); 384 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Expand); 385 } 386 387 if (Subtarget.hasDirectMove() && isPPC64) { 388 setOperationAction(ISD::BITCAST, MVT::f32, Legal); 389 setOperationAction(ISD::BITCAST, MVT::i32, Legal); 390 setOperationAction(ISD::BITCAST, MVT::i64, Legal); 391 setOperationAction(ISD::BITCAST, MVT::f64, Legal); 392 } else { 393 setOperationAction(ISD::BITCAST, MVT::f32, Expand); 394 setOperationAction(ISD::BITCAST, MVT::i32, Expand); 395 setOperationAction(ISD::BITCAST, MVT::i64, Expand); 396 setOperationAction(ISD::BITCAST, MVT::f64, Expand); 397 } 398 399 // We cannot sextinreg(i1). Expand to shifts. 400 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 401 402 // NOTE: EH_SJLJ_SETJMP/_LONGJMP supported here is NOT intended to support 403 // SjLj exception handling but a light-weight setjmp/longjmp replacement to 404 // support continuation, user-level threading, and etc.. As a result, no 405 // other SjLj exception interfaces are implemented and please don't build 406 // your own exception handling based on them. 407 // LLVM/Clang supports zero-cost DWARF exception handling. 408 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom); 409 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom); 410 411 // We want to legalize GlobalAddress and ConstantPool nodes into the 412 // appropriate instructions to materialize the address. 413 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); 414 setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom); 415 setOperationAction(ISD::BlockAddress, MVT::i32, Custom); 416 setOperationAction(ISD::ConstantPool, MVT::i32, Custom); 417 setOperationAction(ISD::JumpTable, MVT::i32, Custom); 418 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); 419 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); 420 setOperationAction(ISD::BlockAddress, MVT::i64, Custom); 421 setOperationAction(ISD::ConstantPool, MVT::i64, Custom); 422 setOperationAction(ISD::JumpTable, MVT::i64, Custom); 423 424 // TRAP is legal. 425 setOperationAction(ISD::TRAP, MVT::Other, Legal); 426 427 // TRAMPOLINE is custom lowered. 428 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom); 429 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom); 430 431 // VASTART needs to be custom lowered to use the VarArgsFrameIndex 432 setOperationAction(ISD::VASTART , MVT::Other, Custom); 433 434 if (Subtarget.isSVR4ABI()) { 435 if (isPPC64) { 436 // VAARG always uses double-word chunks, so promote anything smaller. 437 setOperationAction(ISD::VAARG, MVT::i1, Promote); 438 AddPromotedToType (ISD::VAARG, MVT::i1, MVT::i64); 439 setOperationAction(ISD::VAARG, MVT::i8, Promote); 440 AddPromotedToType (ISD::VAARG, MVT::i8, MVT::i64); 441 setOperationAction(ISD::VAARG, MVT::i16, Promote); 442 AddPromotedToType (ISD::VAARG, MVT::i16, MVT::i64); 443 setOperationAction(ISD::VAARG, MVT::i32, Promote); 444 AddPromotedToType (ISD::VAARG, MVT::i32, MVT::i64); 445 setOperationAction(ISD::VAARG, MVT::Other, Expand); 446 } else { 447 // VAARG is custom lowered with the 32-bit SVR4 ABI. 448 setOperationAction(ISD::VAARG, MVT::Other, Custom); 449 setOperationAction(ISD::VAARG, MVT::i64, Custom); 450 } 451 } else 452 setOperationAction(ISD::VAARG, MVT::Other, Expand); 453 454 if (Subtarget.isSVR4ABI() && !isPPC64) 455 // VACOPY is custom lowered with the 32-bit SVR4 ABI. 456 setOperationAction(ISD::VACOPY , MVT::Other, Custom); 457 else 458 setOperationAction(ISD::VACOPY , MVT::Other, Expand); 459 460 // Use the default implementation. 461 setOperationAction(ISD::VAEND , MVT::Other, Expand); 462 setOperationAction(ISD::STACKSAVE , MVT::Other, Expand); 463 setOperationAction(ISD::STACKRESTORE , MVT::Other, Custom); 464 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32 , Custom); 465 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64 , Custom); 466 setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, MVT::i32, Custom); 467 setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, MVT::i64, Custom); 468 setOperationAction(ISD::EH_DWARF_CFA, MVT::i32, Custom); 469 setOperationAction(ISD::EH_DWARF_CFA, MVT::i64, Custom); 470 471 // We want to custom lower some of our intrinsics. 472 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 473 474 // To handle counter-based loop conditions. 475 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i1, Custom); 476 477 setOperationAction(ISD::INTRINSIC_VOID, MVT::i8, Custom); 478 setOperationAction(ISD::INTRINSIC_VOID, MVT::i16, Custom); 479 setOperationAction(ISD::INTRINSIC_VOID, MVT::i32, Custom); 480 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); 481 482 // Comparisons that require checking two conditions. 483 if (Subtarget.hasSPE()) { 484 setCondCodeAction(ISD::SETO, MVT::f32, Expand); 485 setCondCodeAction(ISD::SETO, MVT::f64, Expand); 486 setCondCodeAction(ISD::SETUO, MVT::f32, Expand); 487 setCondCodeAction(ISD::SETUO, MVT::f64, Expand); 488 } 489 setCondCodeAction(ISD::SETULT, MVT::f32, Expand); 490 setCondCodeAction(ISD::SETULT, MVT::f64, Expand); 491 setCondCodeAction(ISD::SETUGT, MVT::f32, Expand); 492 setCondCodeAction(ISD::SETUGT, MVT::f64, Expand); 493 setCondCodeAction(ISD::SETUEQ, MVT::f32, Expand); 494 setCondCodeAction(ISD::SETUEQ, MVT::f64, Expand); 495 setCondCodeAction(ISD::SETOGE, MVT::f32, Expand); 496 setCondCodeAction(ISD::SETOGE, MVT::f64, Expand); 497 setCondCodeAction(ISD::SETOLE, MVT::f32, Expand); 498 setCondCodeAction(ISD::SETOLE, MVT::f64, Expand); 499 setCondCodeAction(ISD::SETONE, MVT::f32, Expand); 500 setCondCodeAction(ISD::SETONE, MVT::f64, Expand); 501 502 if (Subtarget.has64BitSupport()) { 503 // They also have instructions for converting between i64 and fp. 504 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); 505 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Expand); 506 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); 507 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand); 508 // This is just the low 32 bits of a (signed) fp->i64 conversion. 509 // We cannot do this with Promote because i64 is not a legal type. 510 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); 511 512 if (Subtarget.hasLFIWAX() || Subtarget.isPPC64()) 513 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); 514 } else { 515 // PowerPC does not have FP_TO_UINT on 32-bit implementations. 516 if (Subtarget.hasSPE()) 517 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Legal); 518 else 519 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Expand); 520 } 521 522 // With the instructions enabled under FPCVT, we can do everything. 523 if (Subtarget.hasFPCVT()) { 524 if (Subtarget.has64BitSupport()) { 525 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); 526 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); 527 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); 528 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom); 529 } 530 531 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); 532 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); 533 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); 534 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom); 535 } 536 537 if (Subtarget.use64BitRegs()) { 538 // 64-bit PowerPC implementations can support i64 types directly 539 addRegisterClass(MVT::i64, &PPC::G8RCRegClass); 540 // BUILD_PAIR can't be handled natively, and should be expanded to shl/or 541 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand); 542 // 64-bit PowerPC wants to expand i128 shifts itself. 543 setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom); 544 setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom); 545 setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom); 546 } else { 547 // 32-bit PowerPC wants to expand i64 shifts itself. 548 setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom); 549 setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom); 550 setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom); 551 } 552 553 if (Subtarget.hasAltivec()) { 554 // First set operation action for all vector types to expand. Then we 555 // will selectively turn on ones that can be effectively codegen'd. 556 for (MVT VT : MVT::vector_valuetypes()) { 557 // add/sub are legal for all supported vector VT's. 558 setOperationAction(ISD::ADD, VT, Legal); 559 setOperationAction(ISD::SUB, VT, Legal); 560 561 // For v2i64, these are only valid with P8Vector. This is corrected after 562 // the loop. 563 setOperationAction(ISD::SMAX, VT, Legal); 564 setOperationAction(ISD::SMIN, VT, Legal); 565 setOperationAction(ISD::UMAX, VT, Legal); 566 setOperationAction(ISD::UMIN, VT, Legal); 567 568 if (Subtarget.hasVSX()) { 569 setOperationAction(ISD::FMAXNUM, VT, Legal); 570 setOperationAction(ISD::FMINNUM, VT, Legal); 571 } 572 573 // Vector instructions introduced in P8 574 if (Subtarget.hasP8Altivec() && (VT.SimpleTy != MVT::v1i128)) { 575 setOperationAction(ISD::CTPOP, VT, Legal); 576 setOperationAction(ISD::CTLZ, VT, Legal); 577 } 578 else { 579 setOperationAction(ISD::CTPOP, VT, Expand); 580 setOperationAction(ISD::CTLZ, VT, Expand); 581 } 582 583 // Vector instructions introduced in P9 584 if (Subtarget.hasP9Altivec() && (VT.SimpleTy != MVT::v1i128)) 585 setOperationAction(ISD::CTTZ, VT, Legal); 586 else 587 setOperationAction(ISD::CTTZ, VT, Expand); 588 589 // We promote all shuffles to v16i8. 590 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Promote); 591 AddPromotedToType (ISD::VECTOR_SHUFFLE, VT, MVT::v16i8); 592 593 // We promote all non-typed operations to v4i32. 594 setOperationAction(ISD::AND , VT, Promote); 595 AddPromotedToType (ISD::AND , VT, MVT::v4i32); 596 setOperationAction(ISD::OR , VT, Promote); 597 AddPromotedToType (ISD::OR , VT, MVT::v4i32); 598 setOperationAction(ISD::XOR , VT, Promote); 599 AddPromotedToType (ISD::XOR , VT, MVT::v4i32); 600 setOperationAction(ISD::LOAD , VT, Promote); 601 AddPromotedToType (ISD::LOAD , VT, MVT::v4i32); 602 setOperationAction(ISD::SELECT, VT, Promote); 603 AddPromotedToType (ISD::SELECT, VT, MVT::v4i32); 604 setOperationAction(ISD::VSELECT, VT, Legal); 605 setOperationAction(ISD::SELECT_CC, VT, Promote); 606 AddPromotedToType (ISD::SELECT_CC, VT, MVT::v4i32); 607 setOperationAction(ISD::STORE, VT, Promote); 608 AddPromotedToType (ISD::STORE, VT, MVT::v4i32); 609 610 // No other operations are legal. 611 setOperationAction(ISD::MUL , VT, Expand); 612 setOperationAction(ISD::SDIV, VT, Expand); 613 setOperationAction(ISD::SREM, VT, Expand); 614 setOperationAction(ISD::UDIV, VT, Expand); 615 setOperationAction(ISD::UREM, VT, Expand); 616 setOperationAction(ISD::FDIV, VT, Expand); 617 setOperationAction(ISD::FREM, VT, Expand); 618 setOperationAction(ISD::FNEG, VT, Expand); 619 setOperationAction(ISD::FSQRT, VT, Expand); 620 setOperationAction(ISD::FLOG, VT, Expand); 621 setOperationAction(ISD::FLOG10, VT, Expand); 622 setOperationAction(ISD::FLOG2, VT, Expand); 623 setOperationAction(ISD::FEXP, VT, Expand); 624 setOperationAction(ISD::FEXP2, VT, Expand); 625 setOperationAction(ISD::FSIN, VT, Expand); 626 setOperationAction(ISD::FCOS, VT, Expand); 627 setOperationAction(ISD::FABS, VT, Expand); 628 setOperationAction(ISD::FFLOOR, VT, Expand); 629 setOperationAction(ISD::FCEIL, VT, Expand); 630 setOperationAction(ISD::FTRUNC, VT, Expand); 631 setOperationAction(ISD::FRINT, VT, Expand); 632 setOperationAction(ISD::FNEARBYINT, VT, Expand); 633 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Expand); 634 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand); 635 setOperationAction(ISD::BUILD_VECTOR, VT, Expand); 636 setOperationAction(ISD::MULHU, VT, Expand); 637 setOperationAction(ISD::MULHS, VT, Expand); 638 setOperationAction(ISD::UMUL_LOHI, VT, Expand); 639 setOperationAction(ISD::SMUL_LOHI, VT, Expand); 640 setOperationAction(ISD::UDIVREM, VT, Expand); 641 setOperationAction(ISD::SDIVREM, VT, Expand); 642 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Expand); 643 setOperationAction(ISD::FPOW, VT, Expand); 644 setOperationAction(ISD::BSWAP, VT, Expand); 645 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); 646 setOperationAction(ISD::ROTL, VT, Expand); 647 setOperationAction(ISD::ROTR, VT, Expand); 648 649 for (MVT InnerVT : MVT::vector_valuetypes()) { 650 setTruncStoreAction(VT, InnerVT, Expand); 651 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); 652 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); 653 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); 654 } 655 } 656 if (!Subtarget.hasP8Vector()) { 657 setOperationAction(ISD::SMAX, MVT::v2i64, Expand); 658 setOperationAction(ISD::SMIN, MVT::v2i64, Expand); 659 setOperationAction(ISD::UMAX, MVT::v2i64, Expand); 660 setOperationAction(ISD::UMIN, MVT::v2i64, Expand); 661 } 662 663 for (auto VT : {MVT::v2i64, MVT::v4i32, MVT::v8i16, MVT::v16i8}) 664 setOperationAction(ISD::ABS, VT, Custom); 665 666 // We can custom expand all VECTOR_SHUFFLEs to VPERM, others we can handle 667 // with merges, splats, etc. 668 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16i8, Custom); 669 670 // Vector truncates to sub-word integer that fit in an Altivec/VSX register 671 // are cheap, so handle them before they get expanded to scalar. 672 setOperationAction(ISD::TRUNCATE, MVT::v8i8, Custom); 673 setOperationAction(ISD::TRUNCATE, MVT::v4i8, Custom); 674 setOperationAction(ISD::TRUNCATE, MVT::v2i8, Custom); 675 setOperationAction(ISD::TRUNCATE, MVT::v4i16, Custom); 676 setOperationAction(ISD::TRUNCATE, MVT::v2i16, Custom); 677 678 setOperationAction(ISD::AND , MVT::v4i32, Legal); 679 setOperationAction(ISD::OR , MVT::v4i32, Legal); 680 setOperationAction(ISD::XOR , MVT::v4i32, Legal); 681 setOperationAction(ISD::LOAD , MVT::v4i32, Legal); 682 setOperationAction(ISD::SELECT, MVT::v4i32, 683 Subtarget.useCRBits() ? Legal : Expand); 684 setOperationAction(ISD::STORE , MVT::v4i32, Legal); 685 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal); 686 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal); 687 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal); 688 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal); 689 setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal); 690 setOperationAction(ISD::FCEIL, MVT::v4f32, Legal); 691 setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal); 692 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal); 693 694 // Without hasP8Altivec set, v2i64 SMAX isn't available. 695 // But ABS custom lowering requires SMAX support. 696 if (!Subtarget.hasP8Altivec()) 697 setOperationAction(ISD::ABS, MVT::v2i64, Expand); 698 699 addRegisterClass(MVT::v4f32, &PPC::VRRCRegClass); 700 addRegisterClass(MVT::v4i32, &PPC::VRRCRegClass); 701 addRegisterClass(MVT::v8i16, &PPC::VRRCRegClass); 702 addRegisterClass(MVT::v16i8, &PPC::VRRCRegClass); 703 704 setOperationAction(ISD::MUL, MVT::v4f32, Legal); 705 setOperationAction(ISD::FMA, MVT::v4f32, Legal); 706 707 if (TM.Options.UnsafeFPMath || Subtarget.hasVSX()) { 708 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); 709 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); 710 } 711 712 if (Subtarget.hasP8Altivec()) 713 setOperationAction(ISD::MUL, MVT::v4i32, Legal); 714 else 715 setOperationAction(ISD::MUL, MVT::v4i32, Custom); 716 717 setOperationAction(ISD::MUL, MVT::v8i16, Custom); 718 setOperationAction(ISD::MUL, MVT::v16i8, Custom); 719 720 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Custom); 721 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i32, Custom); 722 723 setOperationAction(ISD::BUILD_VECTOR, MVT::v16i8, Custom); 724 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i16, Custom); 725 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i32, Custom); 726 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); 727 728 // Altivec does not contain unordered floating-point compare instructions 729 setCondCodeAction(ISD::SETUO, MVT::v4f32, Expand); 730 setCondCodeAction(ISD::SETUEQ, MVT::v4f32, Expand); 731 setCondCodeAction(ISD::SETO, MVT::v4f32, Expand); 732 setCondCodeAction(ISD::SETONE, MVT::v4f32, Expand); 733 734 if (Subtarget.hasVSX()) { 735 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal); 736 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Legal); 737 if (Subtarget.hasP8Vector()) { 738 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal); 739 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Legal); 740 } 741 if (Subtarget.hasDirectMove() && isPPC64) { 742 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Legal); 743 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Legal); 744 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i32, Legal); 745 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2i64, Legal); 746 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Legal); 747 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Legal); 748 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Legal); 749 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Legal); 750 } 751 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Legal); 752 753 setOperationAction(ISD::FFLOOR, MVT::v2f64, Legal); 754 setOperationAction(ISD::FCEIL, MVT::v2f64, Legal); 755 setOperationAction(ISD::FTRUNC, MVT::v2f64, Legal); 756 setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Legal); 757 setOperationAction(ISD::FROUND, MVT::v2f64, Legal); 758 759 setOperationAction(ISD::FROUND, MVT::v4f32, Legal); 760 761 setOperationAction(ISD::MUL, MVT::v2f64, Legal); 762 setOperationAction(ISD::FMA, MVT::v2f64, Legal); 763 764 setOperationAction(ISD::FDIV, MVT::v2f64, Legal); 765 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); 766 767 // Share the Altivec comparison restrictions. 768 setCondCodeAction(ISD::SETUO, MVT::v2f64, Expand); 769 setCondCodeAction(ISD::SETUEQ, MVT::v2f64, Expand); 770 setCondCodeAction(ISD::SETO, MVT::v2f64, Expand); 771 setCondCodeAction(ISD::SETONE, MVT::v2f64, Expand); 772 773 setOperationAction(ISD::LOAD, MVT::v2f64, Legal); 774 setOperationAction(ISD::STORE, MVT::v2f64, Legal); 775 776 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Legal); 777 778 if (Subtarget.hasP8Vector()) 779 addRegisterClass(MVT::f32, &PPC::VSSRCRegClass); 780 781 addRegisterClass(MVT::f64, &PPC::VSFRCRegClass); 782 783 addRegisterClass(MVT::v4i32, &PPC::VSRCRegClass); 784 addRegisterClass(MVT::v4f32, &PPC::VSRCRegClass); 785 addRegisterClass(MVT::v2f64, &PPC::VSRCRegClass); 786 787 if (Subtarget.hasP8Altivec()) { 788 setOperationAction(ISD::SHL, MVT::v2i64, Legal); 789 setOperationAction(ISD::SRA, MVT::v2i64, Legal); 790 setOperationAction(ISD::SRL, MVT::v2i64, Legal); 791 792 // 128 bit shifts can be accomplished via 3 instructions for SHL and 793 // SRL, but not for SRA because of the instructions available: 794 // VS{RL} and VS{RL}O. However due to direct move costs, it's not worth 795 // doing 796 setOperationAction(ISD::SHL, MVT::v1i128, Expand); 797 setOperationAction(ISD::SRL, MVT::v1i128, Expand); 798 setOperationAction(ISD::SRA, MVT::v1i128, Expand); 799 800 setOperationAction(ISD::SETCC, MVT::v2i64, Legal); 801 } 802 else { 803 setOperationAction(ISD::SHL, MVT::v2i64, Expand); 804 setOperationAction(ISD::SRA, MVT::v2i64, Expand); 805 setOperationAction(ISD::SRL, MVT::v2i64, Expand); 806 807 setOperationAction(ISD::SETCC, MVT::v2i64, Custom); 808 809 // VSX v2i64 only supports non-arithmetic operations. 810 setOperationAction(ISD::ADD, MVT::v2i64, Expand); 811 setOperationAction(ISD::SUB, MVT::v2i64, Expand); 812 } 813 814 setOperationAction(ISD::LOAD, MVT::v2i64, Promote); 815 AddPromotedToType (ISD::LOAD, MVT::v2i64, MVT::v2f64); 816 setOperationAction(ISD::STORE, MVT::v2i64, Promote); 817 AddPromotedToType (ISD::STORE, MVT::v2i64, MVT::v2f64); 818 819 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Legal); 820 821 setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Legal); 822 setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Legal); 823 setOperationAction(ISD::FP_TO_SINT, MVT::v2i64, Legal); 824 setOperationAction(ISD::FP_TO_UINT, MVT::v2i64, Legal); 825 826 // Custom handling for partial vectors of integers converted to 827 // floating point. We already have optimal handling for v2i32 through 828 // the DAG combine, so those aren't necessary. 829 setOperationAction(ISD::UINT_TO_FP, MVT::v2i8, Custom); 830 setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Custom); 831 setOperationAction(ISD::UINT_TO_FP, MVT::v2i16, Custom); 832 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom); 833 setOperationAction(ISD::SINT_TO_FP, MVT::v2i8, Custom); 834 setOperationAction(ISD::SINT_TO_FP, MVT::v4i8, Custom); 835 setOperationAction(ISD::SINT_TO_FP, MVT::v2i16, Custom); 836 setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom); 837 838 setOperationAction(ISD::FNEG, MVT::v4f32, Legal); 839 setOperationAction(ISD::FNEG, MVT::v2f64, Legal); 840 setOperationAction(ISD::FABS, MVT::v4f32, Legal); 841 setOperationAction(ISD::FABS, MVT::v2f64, Legal); 842 setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Legal); 843 setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Legal); 844 845 if (Subtarget.hasDirectMove()) 846 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); 847 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom); 848 849 addRegisterClass(MVT::v2i64, &PPC::VSRCRegClass); 850 } 851 852 if (Subtarget.hasP8Altivec()) { 853 addRegisterClass(MVT::v2i64, &PPC::VRRCRegClass); 854 addRegisterClass(MVT::v1i128, &PPC::VRRCRegClass); 855 } 856 857 if (Subtarget.hasP9Vector()) { 858 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); 859 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 860 861 // 128 bit shifts can be accomplished via 3 instructions for SHL and 862 // SRL, but not for SRA because of the instructions available: 863 // VS{RL} and VS{RL}O. 864 setOperationAction(ISD::SHL, MVT::v1i128, Legal); 865 setOperationAction(ISD::SRL, MVT::v1i128, Legal); 866 setOperationAction(ISD::SRA, MVT::v1i128, Expand); 867 868 if (EnableQuadPrecision) { 869 addRegisterClass(MVT::f128, &PPC::VRRCRegClass); 870 setOperationAction(ISD::FADD, MVT::f128, Legal); 871 setOperationAction(ISD::FSUB, MVT::f128, Legal); 872 setOperationAction(ISD::FDIV, MVT::f128, Legal); 873 setOperationAction(ISD::FMUL, MVT::f128, Legal); 874 setOperationAction(ISD::FP_EXTEND, MVT::f128, Legal); 875 // No extending loads to f128 on PPC. 876 for (MVT FPT : MVT::fp_valuetypes()) 877 setLoadExtAction(ISD::EXTLOAD, MVT::f128, FPT, Expand); 878 setOperationAction(ISD::FMA, MVT::f128, Legal); 879 setCondCodeAction(ISD::SETULT, MVT::f128, Expand); 880 setCondCodeAction(ISD::SETUGT, MVT::f128, Expand); 881 setCondCodeAction(ISD::SETUEQ, MVT::f128, Expand); 882 setCondCodeAction(ISD::SETOGE, MVT::f128, Expand); 883 setCondCodeAction(ISD::SETOLE, MVT::f128, Expand); 884 setCondCodeAction(ISD::SETONE, MVT::f128, Expand); 885 886 setOperationAction(ISD::FTRUNC, MVT::f128, Legal); 887 setOperationAction(ISD::FRINT, MVT::f128, Legal); 888 setOperationAction(ISD::FFLOOR, MVT::f128, Legal); 889 setOperationAction(ISD::FCEIL, MVT::f128, Legal); 890 setOperationAction(ISD::FNEARBYINT, MVT::f128, Legal); 891 setOperationAction(ISD::FROUND, MVT::f128, Legal); 892 893 setOperationAction(ISD::SELECT, MVT::f128, Expand); 894 setOperationAction(ISD::FP_ROUND, MVT::f64, Legal); 895 setOperationAction(ISD::FP_ROUND, MVT::f32, Legal); 896 setTruncStoreAction(MVT::f128, MVT::f64, Expand); 897 setTruncStoreAction(MVT::f128, MVT::f32, Expand); 898 setOperationAction(ISD::BITCAST, MVT::i128, Custom); 899 // No implementation for these ops for PowerPC. 900 setOperationAction(ISD::FSIN , MVT::f128, Expand); 901 setOperationAction(ISD::FCOS , MVT::f128, Expand); 902 setOperationAction(ISD::FPOW, MVT::f128, Expand); 903 setOperationAction(ISD::FPOWI, MVT::f128, Expand); 904 setOperationAction(ISD::FREM, MVT::f128, Expand); 905 } 906 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom); 907 908 } 909 910 if (Subtarget.hasP9Altivec()) { 911 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 912 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom); 913 } 914 } 915 916 if (Subtarget.hasQPX()) { 917 setOperationAction(ISD::FADD, MVT::v4f64, Legal); 918 setOperationAction(ISD::FSUB, MVT::v4f64, Legal); 919 setOperationAction(ISD::FMUL, MVT::v4f64, Legal); 920 setOperationAction(ISD::FREM, MVT::v4f64, Expand); 921 922 setOperationAction(ISD::FCOPYSIGN, MVT::v4f64, Legal); 923 setOperationAction(ISD::FGETSIGN, MVT::v4f64, Expand); 924 925 setOperationAction(ISD::LOAD , MVT::v4f64, Custom); 926 setOperationAction(ISD::STORE , MVT::v4f64, Custom); 927 928 setTruncStoreAction(MVT::v4f64, MVT::v4f32, Custom); 929 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Custom); 930 931 if (!Subtarget.useCRBits()) 932 setOperationAction(ISD::SELECT, MVT::v4f64, Expand); 933 setOperationAction(ISD::VSELECT, MVT::v4f64, Legal); 934 935 setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4f64, Legal); 936 setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4f64, Expand); 937 setOperationAction(ISD::CONCAT_VECTORS , MVT::v4f64, Expand); 938 setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4f64, Expand); 939 setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4f64, Custom); 940 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f64, Legal); 941 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f64, Custom); 942 943 setOperationAction(ISD::FP_TO_SINT , MVT::v4f64, Legal); 944 setOperationAction(ISD::FP_TO_UINT , MVT::v4f64, Expand); 945 946 setOperationAction(ISD::FP_ROUND , MVT::v4f32, Legal); 947 setOperationAction(ISD::FP_ROUND_INREG , MVT::v4f32, Expand); 948 setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Legal); 949 950 setOperationAction(ISD::FNEG , MVT::v4f64, Legal); 951 setOperationAction(ISD::FABS , MVT::v4f64, Legal); 952 setOperationAction(ISD::FSIN , MVT::v4f64, Expand); 953 setOperationAction(ISD::FCOS , MVT::v4f64, Expand); 954 setOperationAction(ISD::FPOW , MVT::v4f64, Expand); 955 setOperationAction(ISD::FLOG , MVT::v4f64, Expand); 956 setOperationAction(ISD::FLOG2 , MVT::v4f64, Expand); 957 setOperationAction(ISD::FLOG10 , MVT::v4f64, Expand); 958 setOperationAction(ISD::FEXP , MVT::v4f64, Expand); 959 setOperationAction(ISD::FEXP2 , MVT::v4f64, Expand); 960 961 setOperationAction(ISD::FMINNUM, MVT::v4f64, Legal); 962 setOperationAction(ISD::FMAXNUM, MVT::v4f64, Legal); 963 964 setIndexedLoadAction(ISD::PRE_INC, MVT::v4f64, Legal); 965 setIndexedStoreAction(ISD::PRE_INC, MVT::v4f64, Legal); 966 967 addRegisterClass(MVT::v4f64, &PPC::QFRCRegClass); 968 969 setOperationAction(ISD::FADD, MVT::v4f32, Legal); 970 setOperationAction(ISD::FSUB, MVT::v4f32, Legal); 971 setOperationAction(ISD::FMUL, MVT::v4f32, Legal); 972 setOperationAction(ISD::FREM, MVT::v4f32, Expand); 973 974 setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Legal); 975 setOperationAction(ISD::FGETSIGN, MVT::v4f32, Expand); 976 977 setOperationAction(ISD::LOAD , MVT::v4f32, Custom); 978 setOperationAction(ISD::STORE , MVT::v4f32, Custom); 979 980 if (!Subtarget.useCRBits()) 981 setOperationAction(ISD::SELECT, MVT::v4f32, Expand); 982 setOperationAction(ISD::VSELECT, MVT::v4f32, Legal); 983 984 setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4f32, Legal); 985 setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4f32, Expand); 986 setOperationAction(ISD::CONCAT_VECTORS , MVT::v4f32, Expand); 987 setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4f32, Expand); 988 setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4f32, Custom); 989 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal); 990 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); 991 992 setOperationAction(ISD::FP_TO_SINT , MVT::v4f32, Legal); 993 setOperationAction(ISD::FP_TO_UINT , MVT::v4f32, Expand); 994 995 setOperationAction(ISD::FNEG , MVT::v4f32, Legal); 996 setOperationAction(ISD::FABS , MVT::v4f32, Legal); 997 setOperationAction(ISD::FSIN , MVT::v4f32, Expand); 998 setOperationAction(ISD::FCOS , MVT::v4f32, Expand); 999 setOperationAction(ISD::FPOW , MVT::v4f32, Expand); 1000 setOperationAction(ISD::FLOG , MVT::v4f32, Expand); 1001 setOperationAction(ISD::FLOG2 , MVT::v4f32, Expand); 1002 setOperationAction(ISD::FLOG10 , MVT::v4f32, Expand); 1003 setOperationAction(ISD::FEXP , MVT::v4f32, Expand); 1004 setOperationAction(ISD::FEXP2 , MVT::v4f32, Expand); 1005 1006 setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal); 1007 setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal); 1008 1009 setIndexedLoadAction(ISD::PRE_INC, MVT::v4f32, Legal); 1010 setIndexedStoreAction(ISD::PRE_INC, MVT::v4f32, Legal); 1011 1012 addRegisterClass(MVT::v4f32, &PPC::QSRCRegClass); 1013 1014 setOperationAction(ISD::AND , MVT::v4i1, Legal); 1015 setOperationAction(ISD::OR , MVT::v4i1, Legal); 1016 setOperationAction(ISD::XOR , MVT::v4i1, Legal); 1017 1018 if (!Subtarget.useCRBits()) 1019 setOperationAction(ISD::SELECT, MVT::v4i1, Expand); 1020 setOperationAction(ISD::VSELECT, MVT::v4i1, Legal); 1021 1022 setOperationAction(ISD::LOAD , MVT::v4i1, Custom); 1023 setOperationAction(ISD::STORE , MVT::v4i1, Custom); 1024 1025 setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4i1, Custom); 1026 setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4i1, Expand); 1027 setOperationAction(ISD::CONCAT_VECTORS , MVT::v4i1, Expand); 1028 setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4i1, Expand); 1029 setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4i1, Custom); 1030 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i1, Expand); 1031 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i1, Custom); 1032 1033 setOperationAction(ISD::SINT_TO_FP, MVT::v4i1, Custom); 1034 setOperationAction(ISD::UINT_TO_FP, MVT::v4i1, Custom); 1035 1036 addRegisterClass(MVT::v4i1, &PPC::QBRCRegClass); 1037 1038 setOperationAction(ISD::FFLOOR, MVT::v4f64, Legal); 1039 setOperationAction(ISD::FCEIL, MVT::v4f64, Legal); 1040 setOperationAction(ISD::FTRUNC, MVT::v4f64, Legal); 1041 setOperationAction(ISD::FROUND, MVT::v4f64, Legal); 1042 1043 setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal); 1044 setOperationAction(ISD::FCEIL, MVT::v4f32, Legal); 1045 setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal); 1046 setOperationAction(ISD::FROUND, MVT::v4f32, Legal); 1047 1048 setOperationAction(ISD::FNEARBYINT, MVT::v4f64, Expand); 1049 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Expand); 1050 1051 // These need to set FE_INEXACT, and so cannot be vectorized here. 1052 setOperationAction(ISD::FRINT, MVT::v4f64, Expand); 1053 setOperationAction(ISD::FRINT, MVT::v4f32, Expand); 1054 1055 if (TM.Options.UnsafeFPMath) { 1056 setOperationAction(ISD::FDIV, MVT::v4f64, Legal); 1057 setOperationAction(ISD::FSQRT, MVT::v4f64, Legal); 1058 1059 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); 1060 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); 1061 } else { 1062 setOperationAction(ISD::FDIV, MVT::v4f64, Expand); 1063 setOperationAction(ISD::FSQRT, MVT::v4f64, Expand); 1064 1065 setOperationAction(ISD::FDIV, MVT::v4f32, Expand); 1066 setOperationAction(ISD::FSQRT, MVT::v4f32, Expand); 1067 } 1068 } 1069 1070 if (Subtarget.has64BitSupport()) 1071 setOperationAction(ISD::PREFETCH, MVT::Other, Legal); 1072 1073 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, isPPC64 ? Legal : Custom); 1074 1075 if (!isPPC64) { 1076 setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Expand); 1077 setOperationAction(ISD::ATOMIC_STORE, MVT::i64, Expand); 1078 } 1079 1080 setBooleanContents(ZeroOrOneBooleanContent); 1081 1082 if (Subtarget.hasAltivec()) { 1083 // Altivec instructions set fields to all zeros or all ones. 1084 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); 1085 } 1086 1087 if (!isPPC64) { 1088 // These libcalls are not available in 32-bit. 1089 setLibcallName(RTLIB::SHL_I128, nullptr); 1090 setLibcallName(RTLIB::SRL_I128, nullptr); 1091 setLibcallName(RTLIB::SRA_I128, nullptr); 1092 } 1093 1094 setStackPointerRegisterToSaveRestore(isPPC64 ? PPC::X1 : PPC::R1); 1095 1096 // We have target-specific dag combine patterns for the following nodes: 1097 setTargetDAGCombine(ISD::ADD); 1098 setTargetDAGCombine(ISD::SHL); 1099 setTargetDAGCombine(ISD::SRA); 1100 setTargetDAGCombine(ISD::SRL); 1101 setTargetDAGCombine(ISD::MUL); 1102 setTargetDAGCombine(ISD::SINT_TO_FP); 1103 setTargetDAGCombine(ISD::BUILD_VECTOR); 1104 if (Subtarget.hasFPCVT()) 1105 setTargetDAGCombine(ISD::UINT_TO_FP); 1106 setTargetDAGCombine(ISD::LOAD); 1107 setTargetDAGCombine(ISD::STORE); 1108 setTargetDAGCombine(ISD::BR_CC); 1109 if (Subtarget.useCRBits()) 1110 setTargetDAGCombine(ISD::BRCOND); 1111 setTargetDAGCombine(ISD::BSWAP); 1112 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN); 1113 setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN); 1114 setTargetDAGCombine(ISD::INTRINSIC_VOID); 1115 1116 setTargetDAGCombine(ISD::SIGN_EXTEND); 1117 setTargetDAGCombine(ISD::ZERO_EXTEND); 1118 setTargetDAGCombine(ISD::ANY_EXTEND); 1119 1120 setTargetDAGCombine(ISD::TRUNCATE); 1121 1122 if (Subtarget.useCRBits()) { 1123 setTargetDAGCombine(ISD::TRUNCATE); 1124 setTargetDAGCombine(ISD::SETCC); 1125 setTargetDAGCombine(ISD::SELECT_CC); 1126 } 1127 1128 // Use reciprocal estimates. 1129 if (TM.Options.UnsafeFPMath) { 1130 setTargetDAGCombine(ISD::FDIV); 1131 setTargetDAGCombine(ISD::FSQRT); 1132 } 1133 1134 if (Subtarget.hasP9Altivec()) { 1135 setTargetDAGCombine(ISD::ABS); 1136 setTargetDAGCombine(ISD::VSELECT); 1137 } 1138 1139 // Darwin long double math library functions have $LDBL128 appended. 1140 if (Subtarget.isDarwin()) { 1141 setLibcallName(RTLIB::COS_PPCF128, "cosl$LDBL128"); 1142 setLibcallName(RTLIB::POW_PPCF128, "powl$LDBL128"); 1143 setLibcallName(RTLIB::REM_PPCF128, "fmodl$LDBL128"); 1144 setLibcallName(RTLIB::SIN_PPCF128, "sinl$LDBL128"); 1145 setLibcallName(RTLIB::SQRT_PPCF128, "sqrtl$LDBL128"); 1146 setLibcallName(RTLIB::LOG_PPCF128, "logl$LDBL128"); 1147 setLibcallName(RTLIB::LOG2_PPCF128, "log2l$LDBL128"); 1148 setLibcallName(RTLIB::LOG10_PPCF128, "log10l$LDBL128"); 1149 setLibcallName(RTLIB::EXP_PPCF128, "expl$LDBL128"); 1150 setLibcallName(RTLIB::EXP2_PPCF128, "exp2l$LDBL128"); 1151 } 1152 1153 if (EnableQuadPrecision) { 1154 setLibcallName(RTLIB::LOG_F128, "logf128"); 1155 setLibcallName(RTLIB::LOG2_F128, "log2f128"); 1156 setLibcallName(RTLIB::LOG10_F128, "log10f128"); 1157 setLibcallName(RTLIB::EXP_F128, "expf128"); 1158 setLibcallName(RTLIB::EXP2_F128, "exp2f128"); 1159 setLibcallName(RTLIB::SIN_F128, "sinf128"); 1160 setLibcallName(RTLIB::COS_F128, "cosf128"); 1161 setLibcallName(RTLIB::POW_F128, "powf128"); 1162 setLibcallName(RTLIB::FMIN_F128, "fminf128"); 1163 setLibcallName(RTLIB::FMAX_F128, "fmaxf128"); 1164 setLibcallName(RTLIB::POWI_F128, "__powikf2"); 1165 setLibcallName(RTLIB::REM_F128, "fmodf128"); 1166 } 1167 1168 // With 32 condition bits, we don't need to sink (and duplicate) compares 1169 // aggressively in CodeGenPrep. 1170 if (Subtarget.useCRBits()) { 1171 setHasMultipleConditionRegisters(); 1172 setJumpIsExpensive(); 1173 } 1174 1175 setMinFunctionAlignment(2); 1176 if (Subtarget.isDarwin()) 1177 setPrefFunctionAlignment(4); 1178 1179 switch (Subtarget.getDarwinDirective()) { 1180 default: break; 1181 case PPC::DIR_970: 1182 case PPC::DIR_A2: 1183 case PPC::DIR_E500: 1184 case PPC::DIR_E500mc: 1185 case PPC::DIR_E5500: 1186 case PPC::DIR_PWR4: 1187 case PPC::DIR_PWR5: 1188 case PPC::DIR_PWR5X: 1189 case PPC::DIR_PWR6: 1190 case PPC::DIR_PWR6X: 1191 case PPC::DIR_PWR7: 1192 case PPC::DIR_PWR8: 1193 case PPC::DIR_PWR9: 1194 setPrefFunctionAlignment(4); 1195 setPrefLoopAlignment(4); 1196 break; 1197 } 1198 1199 if (Subtarget.enableMachineScheduler()) 1200 setSchedulingPreference(Sched::Source); 1201 else 1202 setSchedulingPreference(Sched::Hybrid); 1203 1204 computeRegisterProperties(STI.getRegisterInfo()); 1205 1206 // The Freescale cores do better with aggressive inlining of memcpy and 1207 // friends. GCC uses same threshold of 128 bytes (= 32 word stores). 1208 if (Subtarget.getDarwinDirective() == PPC::DIR_E500mc || 1209 Subtarget.getDarwinDirective() == PPC::DIR_E5500) { 1210 MaxStoresPerMemset = 32; 1211 MaxStoresPerMemsetOptSize = 16; 1212 MaxStoresPerMemcpy = 32; 1213 MaxStoresPerMemcpyOptSize = 8; 1214 MaxStoresPerMemmove = 32; 1215 MaxStoresPerMemmoveOptSize = 8; 1216 } else if (Subtarget.getDarwinDirective() == PPC::DIR_A2) { 1217 // The A2 also benefits from (very) aggressive inlining of memcpy and 1218 // friends. The overhead of a the function call, even when warm, can be 1219 // over one hundred cycles. 1220 MaxStoresPerMemset = 128; 1221 MaxStoresPerMemcpy = 128; 1222 MaxStoresPerMemmove = 128; 1223 MaxLoadsPerMemcmp = 128; 1224 } else { 1225 MaxLoadsPerMemcmp = 8; 1226 MaxLoadsPerMemcmpOptSize = 4; 1227 } 1228 } 1229 1230 /// getMaxByValAlign - Helper for getByValTypeAlignment to determine 1231 /// the desired ByVal argument alignment. 1232 static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign, 1233 unsigned MaxMaxAlign) { 1234 if (MaxAlign == MaxMaxAlign) 1235 return; 1236 if (VectorType *VTy = dyn_cast<VectorType>(Ty)) { 1237 if (MaxMaxAlign >= 32 && VTy->getBitWidth() >= 256) 1238 MaxAlign = 32; 1239 else if (VTy->getBitWidth() >= 128 && MaxAlign < 16) 1240 MaxAlign = 16; 1241 } else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { 1242 unsigned EltAlign = 0; 1243 getMaxByValAlign(ATy->getElementType(), EltAlign, MaxMaxAlign); 1244 if (EltAlign > MaxAlign) 1245 MaxAlign = EltAlign; 1246 } else if (StructType *STy = dyn_cast<StructType>(Ty)) { 1247 for (auto *EltTy : STy->elements()) { 1248 unsigned EltAlign = 0; 1249 getMaxByValAlign(EltTy, EltAlign, MaxMaxAlign); 1250 if (EltAlign > MaxAlign) 1251 MaxAlign = EltAlign; 1252 if (MaxAlign == MaxMaxAlign) 1253 break; 1254 } 1255 } 1256 } 1257 1258 /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate 1259 /// function arguments in the caller parameter area. 1260 unsigned PPCTargetLowering::getByValTypeAlignment(Type *Ty, 1261 const DataLayout &DL) const { 1262 // Darwin passes everything on 4 byte boundary. 1263 if (Subtarget.isDarwin()) 1264 return 4; 1265 1266 // 16byte and wider vectors are passed on 16byte boundary. 1267 // The rest is 8 on PPC64 and 4 on PPC32 boundary. 1268 unsigned Align = Subtarget.isPPC64() ? 8 : 4; 1269 if (Subtarget.hasAltivec() || Subtarget.hasQPX()) 1270 getMaxByValAlign(Ty, Align, Subtarget.hasQPX() ? 32 : 16); 1271 return Align; 1272 } 1273 1274 bool PPCTargetLowering::useSoftFloat() const { 1275 return Subtarget.useSoftFloat(); 1276 } 1277 1278 bool PPCTargetLowering::hasSPE() const { 1279 return Subtarget.hasSPE(); 1280 } 1281 1282 bool PPCTargetLowering::preferIncOfAddToSubOfNot(EVT VT) const { 1283 return VT.isScalarInteger(); 1284 } 1285 1286 const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const { 1287 switch ((PPCISD::NodeType)Opcode) { 1288 case PPCISD::FIRST_NUMBER: break; 1289 case PPCISD::FSEL: return "PPCISD::FSEL"; 1290 case PPCISD::FCFID: return "PPCISD::FCFID"; 1291 case PPCISD::FCFIDU: return "PPCISD::FCFIDU"; 1292 case PPCISD::FCFIDS: return "PPCISD::FCFIDS"; 1293 case PPCISD::FCFIDUS: return "PPCISD::FCFIDUS"; 1294 case PPCISD::FCTIDZ: return "PPCISD::FCTIDZ"; 1295 case PPCISD::FCTIWZ: return "PPCISD::FCTIWZ"; 1296 case PPCISD::FCTIDUZ: return "PPCISD::FCTIDUZ"; 1297 case PPCISD::FCTIWUZ: return "PPCISD::FCTIWUZ"; 1298 case PPCISD::FP_TO_UINT_IN_VSR: 1299 return "PPCISD::FP_TO_UINT_IN_VSR,"; 1300 case PPCISD::FP_TO_SINT_IN_VSR: 1301 return "PPCISD::FP_TO_SINT_IN_VSR"; 1302 case PPCISD::FRE: return "PPCISD::FRE"; 1303 case PPCISD::FRSQRTE: return "PPCISD::FRSQRTE"; 1304 case PPCISD::STFIWX: return "PPCISD::STFIWX"; 1305 case PPCISD::VMADDFP: return "PPCISD::VMADDFP"; 1306 case PPCISD::VNMSUBFP: return "PPCISD::VNMSUBFP"; 1307 case PPCISD::VPERM: return "PPCISD::VPERM"; 1308 case PPCISD::XXSPLT: return "PPCISD::XXSPLT"; 1309 case PPCISD::VECINSERT: return "PPCISD::VECINSERT"; 1310 case PPCISD::XXREVERSE: return "PPCISD::XXREVERSE"; 1311 case PPCISD::XXPERMDI: return "PPCISD::XXPERMDI"; 1312 case PPCISD::VECSHL: return "PPCISD::VECSHL"; 1313 case PPCISD::CMPB: return "PPCISD::CMPB"; 1314 case PPCISD::Hi: return "PPCISD::Hi"; 1315 case PPCISD::Lo: return "PPCISD::Lo"; 1316 case PPCISD::TOC_ENTRY: return "PPCISD::TOC_ENTRY"; 1317 case PPCISD::ATOMIC_CMP_SWAP_8: return "PPCISD::ATOMIC_CMP_SWAP_8"; 1318 case PPCISD::ATOMIC_CMP_SWAP_16: return "PPCISD::ATOMIC_CMP_SWAP_16"; 1319 case PPCISD::DYNALLOC: return "PPCISD::DYNALLOC"; 1320 case PPCISD::DYNAREAOFFSET: return "PPCISD::DYNAREAOFFSET"; 1321 case PPCISD::GlobalBaseReg: return "PPCISD::GlobalBaseReg"; 1322 case PPCISD::SRL: return "PPCISD::SRL"; 1323 case PPCISD::SRA: return "PPCISD::SRA"; 1324 case PPCISD::SHL: return "PPCISD::SHL"; 1325 case PPCISD::SRA_ADDZE: return "PPCISD::SRA_ADDZE"; 1326 case PPCISD::CALL: return "PPCISD::CALL"; 1327 case PPCISD::CALL_NOP: return "PPCISD::CALL_NOP"; 1328 case PPCISD::MTCTR: return "PPCISD::MTCTR"; 1329 case PPCISD::BCTRL: return "PPCISD::BCTRL"; 1330 case PPCISD::BCTRL_LOAD_TOC: return "PPCISD::BCTRL_LOAD_TOC"; 1331 case PPCISD::RET_FLAG: return "PPCISD::RET_FLAG"; 1332 case PPCISD::READ_TIME_BASE: return "PPCISD::READ_TIME_BASE"; 1333 case PPCISD::EH_SJLJ_SETJMP: return "PPCISD::EH_SJLJ_SETJMP"; 1334 case PPCISD::EH_SJLJ_LONGJMP: return "PPCISD::EH_SJLJ_LONGJMP"; 1335 case PPCISD::MFOCRF: return "PPCISD::MFOCRF"; 1336 case PPCISD::MFVSR: return "PPCISD::MFVSR"; 1337 case PPCISD::MTVSRA: return "PPCISD::MTVSRA"; 1338 case PPCISD::MTVSRZ: return "PPCISD::MTVSRZ"; 1339 case PPCISD::SINT_VEC_TO_FP: return "PPCISD::SINT_VEC_TO_FP"; 1340 case PPCISD::UINT_VEC_TO_FP: return "PPCISD::UINT_VEC_TO_FP"; 1341 case PPCISD::ANDIo_1_EQ_BIT: return "PPCISD::ANDIo_1_EQ_BIT"; 1342 case PPCISD::ANDIo_1_GT_BIT: return "PPCISD::ANDIo_1_GT_BIT"; 1343 case PPCISD::VCMP: return "PPCISD::VCMP"; 1344 case PPCISD::VCMPo: return "PPCISD::VCMPo"; 1345 case PPCISD::LBRX: return "PPCISD::LBRX"; 1346 case PPCISD::STBRX: return "PPCISD::STBRX"; 1347 case PPCISD::LFIWAX: return "PPCISD::LFIWAX"; 1348 case PPCISD::LFIWZX: return "PPCISD::LFIWZX"; 1349 case PPCISD::LXSIZX: return "PPCISD::LXSIZX"; 1350 case PPCISD::STXSIX: return "PPCISD::STXSIX"; 1351 case PPCISD::VEXTS: return "PPCISD::VEXTS"; 1352 case PPCISD::SExtVElems: return "PPCISD::SExtVElems"; 1353 case PPCISD::LXVD2X: return "PPCISD::LXVD2X"; 1354 case PPCISD::STXVD2X: return "PPCISD::STXVD2X"; 1355 case PPCISD::ST_VSR_SCAL_INT: 1356 return "PPCISD::ST_VSR_SCAL_INT"; 1357 case PPCISD::COND_BRANCH: return "PPCISD::COND_BRANCH"; 1358 case PPCISD::BDNZ: return "PPCISD::BDNZ"; 1359 case PPCISD::BDZ: return "PPCISD::BDZ"; 1360 case PPCISD::MFFS: return "PPCISD::MFFS"; 1361 case PPCISD::FADDRTZ: return "PPCISD::FADDRTZ"; 1362 case PPCISD::TC_RETURN: return "PPCISD::TC_RETURN"; 1363 case PPCISD::CR6SET: return "PPCISD::CR6SET"; 1364 case PPCISD::CR6UNSET: return "PPCISD::CR6UNSET"; 1365 case PPCISD::PPC32_GOT: return "PPCISD::PPC32_GOT"; 1366 case PPCISD::PPC32_PICGOT: return "PPCISD::PPC32_PICGOT"; 1367 case PPCISD::ADDIS_GOT_TPREL_HA: return "PPCISD::ADDIS_GOT_TPREL_HA"; 1368 case PPCISD::LD_GOT_TPREL_L: return "PPCISD::LD_GOT_TPREL_L"; 1369 case PPCISD::ADD_TLS: return "PPCISD::ADD_TLS"; 1370 case PPCISD::ADDIS_TLSGD_HA: return "PPCISD::ADDIS_TLSGD_HA"; 1371 case PPCISD::ADDI_TLSGD_L: return "PPCISD::ADDI_TLSGD_L"; 1372 case PPCISD::GET_TLS_ADDR: return "PPCISD::GET_TLS_ADDR"; 1373 case PPCISD::ADDI_TLSGD_L_ADDR: return "PPCISD::ADDI_TLSGD_L_ADDR"; 1374 case PPCISD::ADDIS_TLSLD_HA: return "PPCISD::ADDIS_TLSLD_HA"; 1375 case PPCISD::ADDI_TLSLD_L: return "PPCISD::ADDI_TLSLD_L"; 1376 case PPCISD::GET_TLSLD_ADDR: return "PPCISD::GET_TLSLD_ADDR"; 1377 case PPCISD::ADDI_TLSLD_L_ADDR: return "PPCISD::ADDI_TLSLD_L_ADDR"; 1378 case PPCISD::ADDIS_DTPREL_HA: return "PPCISD::ADDIS_DTPREL_HA"; 1379 case PPCISD::ADDI_DTPREL_L: return "PPCISD::ADDI_DTPREL_L"; 1380 case PPCISD::VADD_SPLAT: return "PPCISD::VADD_SPLAT"; 1381 case PPCISD::SC: return "PPCISD::SC"; 1382 case PPCISD::CLRBHRB: return "PPCISD::CLRBHRB"; 1383 case PPCISD::MFBHRBE: return "PPCISD::MFBHRBE"; 1384 case PPCISD::RFEBB: return "PPCISD::RFEBB"; 1385 case PPCISD::XXSWAPD: return "PPCISD::XXSWAPD"; 1386 case PPCISD::SWAP_NO_CHAIN: return "PPCISD::SWAP_NO_CHAIN"; 1387 case PPCISD::VABSD: return "PPCISD::VABSD"; 1388 case PPCISD::QVFPERM: return "PPCISD::QVFPERM"; 1389 case PPCISD::QVGPCI: return "PPCISD::QVGPCI"; 1390 case PPCISD::QVALIGNI: return "PPCISD::QVALIGNI"; 1391 case PPCISD::QVESPLATI: return "PPCISD::QVESPLATI"; 1392 case PPCISD::QBFLT: return "PPCISD::QBFLT"; 1393 case PPCISD::QVLFSb: return "PPCISD::QVLFSb"; 1394 case PPCISD::BUILD_FP128: return "PPCISD::BUILD_FP128"; 1395 case PPCISD::BUILD_SPE64: return "PPCISD::BUILD_SPE64"; 1396 case PPCISD::EXTRACT_SPE: return "PPCISD::EXTRACT_SPE"; 1397 case PPCISD::EXTSWSLI: return "PPCISD::EXTSWSLI"; 1398 case PPCISD::LD_VSX_LH: return "PPCISD::LD_VSX_LH"; 1399 case PPCISD::FP_EXTEND_LH: return "PPCISD::FP_EXTEND_LH"; 1400 } 1401 return nullptr; 1402 } 1403 1404 EVT PPCTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C, 1405 EVT VT) const { 1406 if (!VT.isVector()) 1407 return Subtarget.useCRBits() ? MVT::i1 : MVT::i32; 1408 1409 if (Subtarget.hasQPX()) 1410 return EVT::getVectorVT(C, MVT::i1, VT.getVectorNumElements()); 1411 1412 return VT.changeVectorElementTypeToInteger(); 1413 } 1414 1415 bool PPCTargetLowering::enableAggressiveFMAFusion(EVT VT) const { 1416 assert(VT.isFloatingPoint() && "Non-floating-point FMA?"); 1417 return true; 1418 } 1419 1420 //===----------------------------------------------------------------------===// 1421 // Node matching predicates, for use by the tblgen matching code. 1422 //===----------------------------------------------------------------------===// 1423 1424 /// isFloatingPointZero - Return true if this is 0.0 or -0.0. 1425 static bool isFloatingPointZero(SDValue Op) { 1426 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Op)) 1427 return CFP->getValueAPF().isZero(); 1428 else if (ISD::isEXTLoad(Op.getNode()) || ISD::isNON_EXTLoad(Op.getNode())) { 1429 // Maybe this has already been legalized into the constant pool? 1430 if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Op.getOperand(1))) 1431 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal())) 1432 return CFP->getValueAPF().isZero(); 1433 } 1434 return false; 1435 } 1436 1437 /// isConstantOrUndef - Op is either an undef node or a ConstantSDNode. Return 1438 /// true if Op is undef or if it matches the specified value. 1439 static bool isConstantOrUndef(int Op, int Val) { 1440 return Op < 0 || Op == Val; 1441 } 1442 1443 /// isVPKUHUMShuffleMask - Return true if this is the shuffle mask for a 1444 /// VPKUHUM instruction. 1445 /// The ShuffleKind distinguishes between big-endian operations with 1446 /// two different inputs (0), either-endian operations with two identical 1447 /// inputs (1), and little-endian operations with two different inputs (2). 1448 /// For the latter, the input operands are swapped (see PPCInstrAltivec.td). 1449 bool PPC::isVPKUHUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind, 1450 SelectionDAG &DAG) { 1451 bool IsLE = DAG.getDataLayout().isLittleEndian(); 1452 if (ShuffleKind == 0) { 1453 if (IsLE) 1454 return false; 1455 for (unsigned i = 0; i != 16; ++i) 1456 if (!isConstantOrUndef(N->getMaskElt(i), i*2+1)) 1457 return false; 1458 } else if (ShuffleKind == 2) { 1459 if (!IsLE) 1460 return false; 1461 for (unsigned i = 0; i != 16; ++i) 1462 if (!isConstantOrUndef(N->getMaskElt(i), i*2)) 1463 return false; 1464 } else if (ShuffleKind == 1) { 1465 unsigned j = IsLE ? 0 : 1; 1466 for (unsigned i = 0; i != 8; ++i) 1467 if (!isConstantOrUndef(N->getMaskElt(i), i*2+j) || 1468 !isConstantOrUndef(N->getMaskElt(i+8), i*2+j)) 1469 return false; 1470 } 1471 return true; 1472 } 1473 1474 /// isVPKUWUMShuffleMask - Return true if this is the shuffle mask for a 1475 /// VPKUWUM instruction. 1476 /// The ShuffleKind distinguishes between big-endian operations with 1477 /// two different inputs (0), either-endian operations with two identical 1478 /// inputs (1), and little-endian operations with two different inputs (2). 1479 /// For the latter, the input operands are swapped (see PPCInstrAltivec.td). 1480 bool PPC::isVPKUWUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind, 1481 SelectionDAG &DAG) { 1482 bool IsLE = DAG.getDataLayout().isLittleEndian(); 1483 if (ShuffleKind == 0) { 1484 if (IsLE) 1485 return false; 1486 for (unsigned i = 0; i != 16; i += 2) 1487 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+2) || 1488 !isConstantOrUndef(N->getMaskElt(i+1), i*2+3)) 1489 return false; 1490 } else if (ShuffleKind == 2) { 1491 if (!IsLE) 1492 return false; 1493 for (unsigned i = 0; i != 16; i += 2) 1494 if (!isConstantOrUndef(N->getMaskElt(i ), i*2) || 1495 !isConstantOrUndef(N->getMaskElt(i+1), i*2+1)) 1496 return false; 1497 } else if (ShuffleKind == 1) { 1498 unsigned j = IsLE ? 0 : 2; 1499 for (unsigned i = 0; i != 8; i += 2) 1500 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+j) || 1501 !isConstantOrUndef(N->getMaskElt(i+1), i*2+j+1) || 1502 !isConstantOrUndef(N->getMaskElt(i+8), i*2+j) || 1503 !isConstantOrUndef(N->getMaskElt(i+9), i*2+j+1)) 1504 return false; 1505 } 1506 return true; 1507 } 1508 1509 /// isVPKUDUMShuffleMask - Return true if this is the shuffle mask for a 1510 /// VPKUDUM instruction, AND the VPKUDUM instruction exists for the 1511 /// current subtarget. 1512 /// 1513 /// The ShuffleKind distinguishes between big-endian operations with 1514 /// two different inputs (0), either-endian operations with two identical 1515 /// inputs (1), and little-endian operations with two different inputs (2). 1516 /// For the latter, the input operands are swapped (see PPCInstrAltivec.td). 1517 bool PPC::isVPKUDUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind, 1518 SelectionDAG &DAG) { 1519 const PPCSubtarget& Subtarget = 1520 static_cast<const PPCSubtarget&>(DAG.getSubtarget()); 1521 if (!Subtarget.hasP8Vector()) 1522 return false; 1523 1524 bool IsLE = DAG.getDataLayout().isLittleEndian(); 1525 if (ShuffleKind == 0) { 1526 if (IsLE) 1527 return false; 1528 for (unsigned i = 0; i != 16; i += 4) 1529 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+4) || 1530 !isConstantOrUndef(N->getMaskElt(i+1), i*2+5) || 1531 !isConstantOrUndef(N->getMaskElt(i+2), i*2+6) || 1532 !isConstantOrUndef(N->getMaskElt(i+3), i*2+7)) 1533 return false; 1534 } else if (ShuffleKind == 2) { 1535 if (!IsLE) 1536 return false; 1537 for (unsigned i = 0; i != 16; i += 4) 1538 if (!isConstantOrUndef(N->getMaskElt(i ), i*2) || 1539 !isConstantOrUndef(N->getMaskElt(i+1), i*2+1) || 1540 !isConstantOrUndef(N->getMaskElt(i+2), i*2+2) || 1541 !isConstantOrUndef(N->getMaskElt(i+3), i*2+3)) 1542 return false; 1543 } else if (ShuffleKind == 1) { 1544 unsigned j = IsLE ? 0 : 4; 1545 for (unsigned i = 0; i != 8; i += 4) 1546 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+j) || 1547 !isConstantOrUndef(N->getMaskElt(i+1), i*2+j+1) || 1548 !isConstantOrUndef(N->getMaskElt(i+2), i*2+j+2) || 1549 !isConstantOrUndef(N->getMaskElt(i+3), i*2+j+3) || 1550 !isConstantOrUndef(N->getMaskElt(i+8), i*2+j) || 1551 !isConstantOrUndef(N->getMaskElt(i+9), i*2+j+1) || 1552 !isConstantOrUndef(N->getMaskElt(i+10), i*2+j+2) || 1553 !isConstantOrUndef(N->getMaskElt(i+11), i*2+j+3)) 1554 return false; 1555 } 1556 return true; 1557 } 1558 1559 /// isVMerge - Common function, used to match vmrg* shuffles. 1560 /// 1561 static bool isVMerge(ShuffleVectorSDNode *N, unsigned UnitSize, 1562 unsigned LHSStart, unsigned RHSStart) { 1563 if (N->getValueType(0) != MVT::v16i8) 1564 return false; 1565 assert((UnitSize == 1 || UnitSize == 2 || UnitSize == 4) && 1566 "Unsupported merge size!"); 1567 1568 for (unsigned i = 0; i != 8/UnitSize; ++i) // Step over units 1569 for (unsigned j = 0; j != UnitSize; ++j) { // Step over bytes within unit 1570 if (!isConstantOrUndef(N->getMaskElt(i*UnitSize*2+j), 1571 LHSStart+j+i*UnitSize) || 1572 !isConstantOrUndef(N->getMaskElt(i*UnitSize*2+UnitSize+j), 1573 RHSStart+j+i*UnitSize)) 1574 return false; 1575 } 1576 return true; 1577 } 1578 1579 /// isVMRGLShuffleMask - Return true if this is a shuffle mask suitable for 1580 /// a VMRGL* instruction with the specified unit size (1,2 or 4 bytes). 1581 /// The ShuffleKind distinguishes between big-endian merges with two 1582 /// different inputs (0), either-endian merges with two identical inputs (1), 1583 /// and little-endian merges with two different inputs (2). For the latter, 1584 /// the input operands are swapped (see PPCInstrAltivec.td). 1585 bool PPC::isVMRGLShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize, 1586 unsigned ShuffleKind, SelectionDAG &DAG) { 1587 if (DAG.getDataLayout().isLittleEndian()) { 1588 if (ShuffleKind == 1) // unary 1589 return isVMerge(N, UnitSize, 0, 0); 1590 else if (ShuffleKind == 2) // swapped 1591 return isVMerge(N, UnitSize, 0, 16); 1592 else 1593 return false; 1594 } else { 1595 if (ShuffleKind == 1) // unary 1596 return isVMerge(N, UnitSize, 8, 8); 1597 else if (ShuffleKind == 0) // normal 1598 return isVMerge(N, UnitSize, 8, 24); 1599 else 1600 return false; 1601 } 1602 } 1603 1604 /// isVMRGHShuffleMask - Return true if this is a shuffle mask suitable for 1605 /// a VMRGH* instruction with the specified unit size (1,2 or 4 bytes). 1606 /// The ShuffleKind distinguishes between big-endian merges with two 1607 /// different inputs (0), either-endian merges with two identical inputs (1), 1608 /// and little-endian merges with two different inputs (2). For the latter, 1609 /// the input operands are swapped (see PPCInstrAltivec.td). 1610 bool PPC::isVMRGHShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize, 1611 unsigned ShuffleKind, SelectionDAG &DAG) { 1612 if (DAG.getDataLayout().isLittleEndian()) { 1613 if (ShuffleKind == 1) // unary 1614 return isVMerge(N, UnitSize, 8, 8); 1615 else if (ShuffleKind == 2) // swapped 1616 return isVMerge(N, UnitSize, 8, 24); 1617 else 1618 return false; 1619 } else { 1620 if (ShuffleKind == 1) // unary 1621 return isVMerge(N, UnitSize, 0, 0); 1622 else if (ShuffleKind == 0) // normal 1623 return isVMerge(N, UnitSize, 0, 16); 1624 else 1625 return false; 1626 } 1627 } 1628 1629 /** 1630 * Common function used to match vmrgew and vmrgow shuffles 1631 * 1632 * The indexOffset determines whether to look for even or odd words in 1633 * the shuffle mask. This is based on the of the endianness of the target 1634 * machine. 1635 * - Little Endian: 1636 * - Use offset of 0 to check for odd elements 1637 * - Use offset of 4 to check for even elements 1638 * - Big Endian: 1639 * - Use offset of 0 to check for even elements 1640 * - Use offset of 4 to check for odd elements 1641 * A detailed description of the vector element ordering for little endian and 1642 * big endian can be found at 1643 * http://www.ibm.com/developerworks/library/l-ibm-xl-c-cpp-compiler/index.html 1644 * Targeting your applications - what little endian and big endian IBM XL C/C++ 1645 * compiler differences mean to you 1646 * 1647 * The mask to the shuffle vector instruction specifies the indices of the 1648 * elements from the two input vectors to place in the result. The elements are 1649 * numbered in array-access order, starting with the first vector. These vectors 1650 * are always of type v16i8, thus each vector will contain 16 elements of size 1651 * 8. More info on the shuffle vector can be found in the 1652 * http://llvm.org/docs/LangRef.html#shufflevector-instruction 1653 * Language Reference. 1654 * 1655 * The RHSStartValue indicates whether the same input vectors are used (unary) 1656 * or two different input vectors are used, based on the following: 1657 * - If the instruction uses the same vector for both inputs, the range of the 1658 * indices will be 0 to 15. In this case, the RHSStart value passed should 1659 * be 0. 1660 * - If the instruction has two different vectors then the range of the 1661 * indices will be 0 to 31. In this case, the RHSStart value passed should 1662 * be 16 (indices 0-15 specify elements in the first vector while indices 16 1663 * to 31 specify elements in the second vector). 1664 * 1665 * \param[in] N The shuffle vector SD Node to analyze 1666 * \param[in] IndexOffset Specifies whether to look for even or odd elements 1667 * \param[in] RHSStartValue Specifies the starting index for the righthand input 1668 * vector to the shuffle_vector instruction 1669 * \return true iff this shuffle vector represents an even or odd word merge 1670 */ 1671 static bool isVMerge(ShuffleVectorSDNode *N, unsigned IndexOffset, 1672 unsigned RHSStartValue) { 1673 if (N->getValueType(0) != MVT::v16i8) 1674 return false; 1675 1676 for (unsigned i = 0; i < 2; ++i) 1677 for (unsigned j = 0; j < 4; ++j) 1678 if (!isConstantOrUndef(N->getMaskElt(i*4+j), 1679 i*RHSStartValue+j+IndexOffset) || 1680 !isConstantOrUndef(N->getMaskElt(i*4+j+8), 1681 i*RHSStartValue+j+IndexOffset+8)) 1682 return false; 1683 return true; 1684 } 1685 1686 /** 1687 * Determine if the specified shuffle mask is suitable for the vmrgew or 1688 * vmrgow instructions. 1689 * 1690 * \param[in] N The shuffle vector SD Node to analyze 1691 * \param[in] CheckEven Check for an even merge (true) or an odd merge (false) 1692 * \param[in] ShuffleKind Identify the type of merge: 1693 * - 0 = big-endian merge with two different inputs; 1694 * - 1 = either-endian merge with two identical inputs; 1695 * - 2 = little-endian merge with two different inputs (inputs are swapped for 1696 * little-endian merges). 1697 * \param[in] DAG The current SelectionDAG 1698 * \return true iff this shuffle mask 1699 */ 1700 bool PPC::isVMRGEOShuffleMask(ShuffleVectorSDNode *N, bool CheckEven, 1701 unsigned ShuffleKind, SelectionDAG &DAG) { 1702 if (DAG.getDataLayout().isLittleEndian()) { 1703 unsigned indexOffset = CheckEven ? 4 : 0; 1704 if (ShuffleKind == 1) // Unary 1705 return isVMerge(N, indexOffset, 0); 1706 else if (ShuffleKind == 2) // swapped 1707 return isVMerge(N, indexOffset, 16); 1708 else 1709 return false; 1710 } 1711 else { 1712 unsigned indexOffset = CheckEven ? 0 : 4; 1713 if (ShuffleKind == 1) // Unary 1714 return isVMerge(N, indexOffset, 0); 1715 else if (ShuffleKind == 0) // Normal 1716 return isVMerge(N, indexOffset, 16); 1717 else 1718 return false; 1719 } 1720 return false; 1721 } 1722 1723 /// isVSLDOIShuffleMask - If this is a vsldoi shuffle mask, return the shift 1724 /// amount, otherwise return -1. 1725 /// The ShuffleKind distinguishes between big-endian operations with two 1726 /// different inputs (0), either-endian operations with two identical inputs 1727 /// (1), and little-endian operations with two different inputs (2). For the 1728 /// latter, the input operands are swapped (see PPCInstrAltivec.td). 1729 int PPC::isVSLDOIShuffleMask(SDNode *N, unsigned ShuffleKind, 1730 SelectionDAG &DAG) { 1731 if (N->getValueType(0) != MVT::v16i8) 1732 return -1; 1733 1734 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 1735 1736 // Find the first non-undef value in the shuffle mask. 1737 unsigned i; 1738 for (i = 0; i != 16 && SVOp->getMaskElt(i) < 0; ++i) 1739 /*search*/; 1740 1741 if (i == 16) return -1; // all undef. 1742 1743 // Otherwise, check to see if the rest of the elements are consecutively 1744 // numbered from this value. 1745 unsigned ShiftAmt = SVOp->getMaskElt(i); 1746 if (ShiftAmt < i) return -1; 1747 1748 ShiftAmt -= i; 1749 bool isLE = DAG.getDataLayout().isLittleEndian(); 1750 1751 if ((ShuffleKind == 0 && !isLE) || (ShuffleKind == 2 && isLE)) { 1752 // Check the rest of the elements to see if they are consecutive. 1753 for (++i; i != 16; ++i) 1754 if (!isConstantOrUndef(SVOp->getMaskElt(i), ShiftAmt+i)) 1755 return -1; 1756 } else if (ShuffleKind == 1) { 1757 // Check the rest of the elements to see if they are consecutive. 1758 for (++i; i != 16; ++i) 1759 if (!isConstantOrUndef(SVOp->getMaskElt(i), (ShiftAmt+i) & 15)) 1760 return -1; 1761 } else 1762 return -1; 1763 1764 if (isLE) 1765 ShiftAmt = 16 - ShiftAmt; 1766 1767 return ShiftAmt; 1768 } 1769 1770 /// isSplatShuffleMask - Return true if the specified VECTOR_SHUFFLE operand 1771 /// specifies a splat of a single element that is suitable for input to 1772 /// VSPLTB/VSPLTH/VSPLTW. 1773 bool PPC::isSplatShuffleMask(ShuffleVectorSDNode *N, unsigned EltSize) { 1774 assert(N->getValueType(0) == MVT::v16i8 && 1775 (EltSize == 1 || EltSize == 2 || EltSize == 4)); 1776 1777 // The consecutive indices need to specify an element, not part of two 1778 // different elements. So abandon ship early if this isn't the case. 1779 if (N->getMaskElt(0) % EltSize != 0) 1780 return false; 1781 1782 // This is a splat operation if each element of the permute is the same, and 1783 // if the value doesn't reference the second vector. 1784 unsigned ElementBase = N->getMaskElt(0); 1785 1786 // FIXME: Handle UNDEF elements too! 1787 if (ElementBase >= 16) 1788 return false; 1789 1790 // Check that the indices are consecutive, in the case of a multi-byte element 1791 // splatted with a v16i8 mask. 1792 for (unsigned i = 1; i != EltSize; ++i) 1793 if (N->getMaskElt(i) < 0 || N->getMaskElt(i) != (int)(i+ElementBase)) 1794 return false; 1795 1796 for (unsigned i = EltSize, e = 16; i != e; i += EltSize) { 1797 if (N->getMaskElt(i) < 0) continue; 1798 for (unsigned j = 0; j != EltSize; ++j) 1799 if (N->getMaskElt(i+j) != N->getMaskElt(j)) 1800 return false; 1801 } 1802 return true; 1803 } 1804 1805 /// Check that the mask is shuffling N byte elements. Within each N byte 1806 /// element of the mask, the indices could be either in increasing or 1807 /// decreasing order as long as they are consecutive. 1808 /// \param[in] N the shuffle vector SD Node to analyze 1809 /// \param[in] Width the element width in bytes, could be 2/4/8/16 (HalfWord/ 1810 /// Word/DoubleWord/QuadWord). 1811 /// \param[in] StepLen the delta indices number among the N byte element, if 1812 /// the mask is in increasing/decreasing order then it is 1/-1. 1813 /// \return true iff the mask is shuffling N byte elements. 1814 static bool isNByteElemShuffleMask(ShuffleVectorSDNode *N, unsigned Width, 1815 int StepLen) { 1816 assert((Width == 2 || Width == 4 || Width == 8 || Width == 16) && 1817 "Unexpected element width."); 1818 assert((StepLen == 1 || StepLen == -1) && "Unexpected element width."); 1819 1820 unsigned NumOfElem = 16 / Width; 1821 unsigned MaskVal[16]; // Width is never greater than 16 1822 for (unsigned i = 0; i < NumOfElem; ++i) { 1823 MaskVal[0] = N->getMaskElt(i * Width); 1824 if ((StepLen == 1) && (MaskVal[0] % Width)) { 1825 return false; 1826 } else if ((StepLen == -1) && ((MaskVal[0] + 1) % Width)) { 1827 return false; 1828 } 1829 1830 for (unsigned int j = 1; j < Width; ++j) { 1831 MaskVal[j] = N->getMaskElt(i * Width + j); 1832 if (MaskVal[j] != MaskVal[j-1] + StepLen) { 1833 return false; 1834 } 1835 } 1836 } 1837 1838 return true; 1839 } 1840 1841 bool PPC::isXXINSERTWMask(ShuffleVectorSDNode *N, unsigned &ShiftElts, 1842 unsigned &InsertAtByte, bool &Swap, bool IsLE) { 1843 if (!isNByteElemShuffleMask(N, 4, 1)) 1844 return false; 1845 1846 // Now we look at mask elements 0,4,8,12 1847 unsigned M0 = N->getMaskElt(0) / 4; 1848 unsigned M1 = N->getMaskElt(4) / 4; 1849 unsigned M2 = N->getMaskElt(8) / 4; 1850 unsigned M3 = N->getMaskElt(12) / 4; 1851 unsigned LittleEndianShifts[] = { 2, 1, 0, 3 }; 1852 unsigned BigEndianShifts[] = { 3, 0, 1, 2 }; 1853 1854 // Below, let H and L be arbitrary elements of the shuffle mask 1855 // where H is in the range [4,7] and L is in the range [0,3]. 1856 // H, 1, 2, 3 or L, 5, 6, 7 1857 if ((M0 > 3 && M1 == 1 && M2 == 2 && M3 == 3) || 1858 (M0 < 4 && M1 == 5 && M2 == 6 && M3 == 7)) { 1859 ShiftElts = IsLE ? LittleEndianShifts[M0 & 0x3] : BigEndianShifts[M0 & 0x3]; 1860 InsertAtByte = IsLE ? 12 : 0; 1861 Swap = M0 < 4; 1862 return true; 1863 } 1864 // 0, H, 2, 3 or 4, L, 6, 7 1865 if ((M1 > 3 && M0 == 0 && M2 == 2 && M3 == 3) || 1866 (M1 < 4 && M0 == 4 && M2 == 6 && M3 == 7)) { 1867 ShiftElts = IsLE ? LittleEndianShifts[M1 & 0x3] : BigEndianShifts[M1 & 0x3]; 1868 InsertAtByte = IsLE ? 8 : 4; 1869 Swap = M1 < 4; 1870 return true; 1871 } 1872 // 0, 1, H, 3 or 4, 5, L, 7 1873 if ((M2 > 3 && M0 == 0 && M1 == 1 && M3 == 3) || 1874 (M2 < 4 && M0 == 4 && M1 == 5 && M3 == 7)) { 1875 ShiftElts = IsLE ? LittleEndianShifts[M2 & 0x3] : BigEndianShifts[M2 & 0x3]; 1876 InsertAtByte = IsLE ? 4 : 8; 1877 Swap = M2 < 4; 1878 return true; 1879 } 1880 // 0, 1, 2, H or 4, 5, 6, L 1881 if ((M3 > 3 && M0 == 0 && M1 == 1 && M2 == 2) || 1882 (M3 < 4 && M0 == 4 && M1 == 5 && M2 == 6)) { 1883 ShiftElts = IsLE ? LittleEndianShifts[M3 & 0x3] : BigEndianShifts[M3 & 0x3]; 1884 InsertAtByte = IsLE ? 0 : 12; 1885 Swap = M3 < 4; 1886 return true; 1887 } 1888 1889 // If both vector operands for the shuffle are the same vector, the mask will 1890 // contain only elements from the first one and the second one will be undef. 1891 if (N->getOperand(1).isUndef()) { 1892 ShiftElts = 0; 1893 Swap = true; 1894 unsigned XXINSERTWSrcElem = IsLE ? 2 : 1; 1895 if (M0 == XXINSERTWSrcElem && M1 == 1 && M2 == 2 && M3 == 3) { 1896 InsertAtByte = IsLE ? 12 : 0; 1897 return true; 1898 } 1899 if (M0 == 0 && M1 == XXINSERTWSrcElem && M2 == 2 && M3 == 3) { 1900 InsertAtByte = IsLE ? 8 : 4; 1901 return true; 1902 } 1903 if (M0 == 0 && M1 == 1 && M2 == XXINSERTWSrcElem && M3 == 3) { 1904 InsertAtByte = IsLE ? 4 : 8; 1905 return true; 1906 } 1907 if (M0 == 0 && M1 == 1 && M2 == 2 && M3 == XXINSERTWSrcElem) { 1908 InsertAtByte = IsLE ? 0 : 12; 1909 return true; 1910 } 1911 } 1912 1913 return false; 1914 } 1915 1916 bool PPC::isXXSLDWIShuffleMask(ShuffleVectorSDNode *N, unsigned &ShiftElts, 1917 bool &Swap, bool IsLE) { 1918 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8"); 1919 // Ensure each byte index of the word is consecutive. 1920 if (!isNByteElemShuffleMask(N, 4, 1)) 1921 return false; 1922 1923 // Now we look at mask elements 0,4,8,12, which are the beginning of words. 1924 unsigned M0 = N->getMaskElt(0) / 4; 1925 unsigned M1 = N->getMaskElt(4) / 4; 1926 unsigned M2 = N->getMaskElt(8) / 4; 1927 unsigned M3 = N->getMaskElt(12) / 4; 1928 1929 // If both vector operands for the shuffle are the same vector, the mask will 1930 // contain only elements from the first one and the second one will be undef. 1931 if (N->getOperand(1).isUndef()) { 1932 assert(M0 < 4 && "Indexing into an undef vector?"); 1933 if (M1 != (M0 + 1) % 4 || M2 != (M1 + 1) % 4 || M3 != (M2 + 1) % 4) 1934 return false; 1935 1936 ShiftElts = IsLE ? (4 - M0) % 4 : M0; 1937 Swap = false; 1938 return true; 1939 } 1940 1941 // Ensure each word index of the ShuffleVector Mask is consecutive. 1942 if (M1 != (M0 + 1) % 8 || M2 != (M1 + 1) % 8 || M3 != (M2 + 1) % 8) 1943 return false; 1944 1945 if (IsLE) { 1946 if (M0 == 0 || M0 == 7 || M0 == 6 || M0 == 5) { 1947 // Input vectors don't need to be swapped if the leading element 1948 // of the result is one of the 3 left elements of the second vector 1949 // (or if there is no shift to be done at all). 1950 Swap = false; 1951 ShiftElts = (8 - M0) % 8; 1952 } else if (M0 == 4 || M0 == 3 || M0 == 2 || M0 == 1) { 1953 // Input vectors need to be swapped if the leading element 1954 // of the result is one of the 3 left elements of the first vector 1955 // (or if we're shifting by 4 - thereby simply swapping the vectors). 1956 Swap = true; 1957 ShiftElts = (4 - M0) % 4; 1958 } 1959 1960 return true; 1961 } else { // BE 1962 if (M0 == 0 || M0 == 1 || M0 == 2 || M0 == 3) { 1963 // Input vectors don't need to be swapped if the leading element 1964 // of the result is one of the 4 elements of the first vector. 1965 Swap = false; 1966 ShiftElts = M0; 1967 } else if (M0 == 4 || M0 == 5 || M0 == 6 || M0 == 7) { 1968 // Input vectors need to be swapped if the leading element 1969 // of the result is one of the 4 elements of the right vector. 1970 Swap = true; 1971 ShiftElts = M0 - 4; 1972 } 1973 1974 return true; 1975 } 1976 } 1977 1978 bool static isXXBRShuffleMaskHelper(ShuffleVectorSDNode *N, int Width) { 1979 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8"); 1980 1981 if (!isNByteElemShuffleMask(N, Width, -1)) 1982 return false; 1983 1984 for (int i = 0; i < 16; i += Width) 1985 if (N->getMaskElt(i) != i + Width - 1) 1986 return false; 1987 1988 return true; 1989 } 1990 1991 bool PPC::isXXBRHShuffleMask(ShuffleVectorSDNode *N) { 1992 return isXXBRShuffleMaskHelper(N, 2); 1993 } 1994 1995 bool PPC::isXXBRWShuffleMask(ShuffleVectorSDNode *N) { 1996 return isXXBRShuffleMaskHelper(N, 4); 1997 } 1998 1999 bool PPC::isXXBRDShuffleMask(ShuffleVectorSDNode *N) { 2000 return isXXBRShuffleMaskHelper(N, 8); 2001 } 2002 2003 bool PPC::isXXBRQShuffleMask(ShuffleVectorSDNode *N) { 2004 return isXXBRShuffleMaskHelper(N, 16); 2005 } 2006 2007 /// Can node \p N be lowered to an XXPERMDI instruction? If so, set \p Swap 2008 /// if the inputs to the instruction should be swapped and set \p DM to the 2009 /// value for the immediate. 2010 /// Specifically, set \p Swap to true only if \p N can be lowered to XXPERMDI 2011 /// AND element 0 of the result comes from the first input (LE) or second input 2012 /// (BE). Set \p DM to the calculated result (0-3) only if \p N can be lowered. 2013 /// \return true iff the given mask of shuffle node \p N is a XXPERMDI shuffle 2014 /// mask. 2015 bool PPC::isXXPERMDIShuffleMask(ShuffleVectorSDNode *N, unsigned &DM, 2016 bool &Swap, bool IsLE) { 2017 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8"); 2018 2019 // Ensure each byte index of the double word is consecutive. 2020 if (!isNByteElemShuffleMask(N, 8, 1)) 2021 return false; 2022 2023 unsigned M0 = N->getMaskElt(0) / 8; 2024 unsigned M1 = N->getMaskElt(8) / 8; 2025 assert(((M0 | M1) < 4) && "A mask element out of bounds?"); 2026 2027 // If both vector operands for the shuffle are the same vector, the mask will 2028 // contain only elements from the first one and the second one will be undef. 2029 if (N->getOperand(1).isUndef()) { 2030 if ((M0 | M1) < 2) { 2031 DM = IsLE ? (((~M1) & 1) << 1) + ((~M0) & 1) : (M0 << 1) + (M1 & 1); 2032 Swap = false; 2033 return true; 2034 } else 2035 return false; 2036 } 2037 2038 if (IsLE) { 2039 if (M0 > 1 && M1 < 2) { 2040 Swap = false; 2041 } else if (M0 < 2 && M1 > 1) { 2042 M0 = (M0 + 2) % 4; 2043 M1 = (M1 + 2) % 4; 2044 Swap = true; 2045 } else 2046 return false; 2047 2048 // Note: if control flow comes here that means Swap is already set above 2049 DM = (((~M1) & 1) << 1) + ((~M0) & 1); 2050 return true; 2051 } else { // BE 2052 if (M0 < 2 && M1 > 1) { 2053 Swap = false; 2054 } else if (M0 > 1 && M1 < 2) { 2055 M0 = (M0 + 2) % 4; 2056 M1 = (M1 + 2) % 4; 2057 Swap = true; 2058 } else 2059 return false; 2060 2061 // Note: if control flow comes here that means Swap is already set above 2062 DM = (M0 << 1) + (M1 & 1); 2063 return true; 2064 } 2065 } 2066 2067 2068 /// getVSPLTImmediate - Return the appropriate VSPLT* immediate to splat the 2069 /// specified isSplatShuffleMask VECTOR_SHUFFLE mask. 2070 unsigned PPC::getVSPLTImmediate(SDNode *N, unsigned EltSize, 2071 SelectionDAG &DAG) { 2072 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 2073 assert(isSplatShuffleMask(SVOp, EltSize)); 2074 if (DAG.getDataLayout().isLittleEndian()) 2075 return (16 / EltSize) - 1 - (SVOp->getMaskElt(0) / EltSize); 2076 else 2077 return SVOp->getMaskElt(0) / EltSize; 2078 } 2079 2080 /// get_VSPLTI_elt - If this is a build_vector of constants which can be formed 2081 /// by using a vspltis[bhw] instruction of the specified element size, return 2082 /// the constant being splatted. The ByteSize field indicates the number of 2083 /// bytes of each element [124] -> [bhw]. 2084 SDValue PPC::get_VSPLTI_elt(SDNode *N, unsigned ByteSize, SelectionDAG &DAG) { 2085 SDValue OpVal(nullptr, 0); 2086 2087 // If ByteSize of the splat is bigger than the element size of the 2088 // build_vector, then we have a case where we are checking for a splat where 2089 // multiple elements of the buildvector are folded together into a single 2090 // logical element of the splat (e.g. "vsplish 1" to splat {0,1}*8). 2091 unsigned EltSize = 16/N->getNumOperands(); 2092 if (EltSize < ByteSize) { 2093 unsigned Multiple = ByteSize/EltSize; // Number of BV entries per spltval. 2094 SDValue UniquedVals[4]; 2095 assert(Multiple > 1 && Multiple <= 4 && "How can this happen?"); 2096 2097 // See if all of the elements in the buildvector agree across. 2098 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 2099 if (N->getOperand(i).isUndef()) continue; 2100 // If the element isn't a constant, bail fully out. 2101 if (!isa<ConstantSDNode>(N->getOperand(i))) return SDValue(); 2102 2103 if (!UniquedVals[i&(Multiple-1)].getNode()) 2104 UniquedVals[i&(Multiple-1)] = N->getOperand(i); 2105 else if (UniquedVals[i&(Multiple-1)] != N->getOperand(i)) 2106 return SDValue(); // no match. 2107 } 2108 2109 // Okay, if we reached this point, UniquedVals[0..Multiple-1] contains 2110 // either constant or undef values that are identical for each chunk. See 2111 // if these chunks can form into a larger vspltis*. 2112 2113 // Check to see if all of the leading entries are either 0 or -1. If 2114 // neither, then this won't fit into the immediate field. 2115 bool LeadingZero = true; 2116 bool LeadingOnes = true; 2117 for (unsigned i = 0; i != Multiple-1; ++i) { 2118 if (!UniquedVals[i].getNode()) continue; // Must have been undefs. 2119 2120 LeadingZero &= isNullConstant(UniquedVals[i]); 2121 LeadingOnes &= isAllOnesConstant(UniquedVals[i]); 2122 } 2123 // Finally, check the least significant entry. 2124 if (LeadingZero) { 2125 if (!UniquedVals[Multiple-1].getNode()) 2126 return DAG.getTargetConstant(0, SDLoc(N), MVT::i32); // 0,0,0,undef 2127 int Val = cast<ConstantSDNode>(UniquedVals[Multiple-1])->getZExtValue(); 2128 if (Val < 16) // 0,0,0,4 -> vspltisw(4) 2129 return DAG.getTargetConstant(Val, SDLoc(N), MVT::i32); 2130 } 2131 if (LeadingOnes) { 2132 if (!UniquedVals[Multiple-1].getNode()) 2133 return DAG.getTargetConstant(~0U, SDLoc(N), MVT::i32); // -1,-1,-1,undef 2134 int Val =cast<ConstantSDNode>(UniquedVals[Multiple-1])->getSExtValue(); 2135 if (Val >= -16) // -1,-1,-1,-2 -> vspltisw(-2) 2136 return DAG.getTargetConstant(Val, SDLoc(N), MVT::i32); 2137 } 2138 2139 return SDValue(); 2140 } 2141 2142 // Check to see if this buildvec has a single non-undef value in its elements. 2143 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 2144 if (N->getOperand(i).isUndef()) continue; 2145 if (!OpVal.getNode()) 2146 OpVal = N->getOperand(i); 2147 else if (OpVal != N->getOperand(i)) 2148 return SDValue(); 2149 } 2150 2151 if (!OpVal.getNode()) return SDValue(); // All UNDEF: use implicit def. 2152 2153 unsigned ValSizeInBytes = EltSize; 2154 uint64_t Value = 0; 2155 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) { 2156 Value = CN->getZExtValue(); 2157 } else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal)) { 2158 assert(CN->getValueType(0) == MVT::f32 && "Only one legal FP vector type!"); 2159 Value = FloatToBits(CN->getValueAPF().convertToFloat()); 2160 } 2161 2162 // If the splat value is larger than the element value, then we can never do 2163 // this splat. The only case that we could fit the replicated bits into our 2164 // immediate field for would be zero, and we prefer to use vxor for it. 2165 if (ValSizeInBytes < ByteSize) return SDValue(); 2166 2167 // If the element value is larger than the splat value, check if it consists 2168 // of a repeated bit pattern of size ByteSize. 2169 if (!APInt(ValSizeInBytes * 8, Value).isSplat(ByteSize * 8)) 2170 return SDValue(); 2171 2172 // Properly sign extend the value. 2173 int MaskVal = SignExtend32(Value, ByteSize * 8); 2174 2175 // If this is zero, don't match, zero matches ISD::isBuildVectorAllZeros. 2176 if (MaskVal == 0) return SDValue(); 2177 2178 // Finally, if this value fits in a 5 bit sext field, return it 2179 if (SignExtend32<5>(MaskVal) == MaskVal) 2180 return DAG.getTargetConstant(MaskVal, SDLoc(N), MVT::i32); 2181 return SDValue(); 2182 } 2183 2184 /// isQVALIGNIShuffleMask - If this is a qvaligni shuffle mask, return the shift 2185 /// amount, otherwise return -1. 2186 int PPC::isQVALIGNIShuffleMask(SDNode *N) { 2187 EVT VT = N->getValueType(0); 2188 if (VT != MVT::v4f64 && VT != MVT::v4f32 && VT != MVT::v4i1) 2189 return -1; 2190 2191 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 2192 2193 // Find the first non-undef value in the shuffle mask. 2194 unsigned i; 2195 for (i = 0; i != 4 && SVOp->getMaskElt(i) < 0; ++i) 2196 /*search*/; 2197 2198 if (i == 4) return -1; // all undef. 2199 2200 // Otherwise, check to see if the rest of the elements are consecutively 2201 // numbered from this value. 2202 unsigned ShiftAmt = SVOp->getMaskElt(i); 2203 if (ShiftAmt < i) return -1; 2204 ShiftAmt -= i; 2205 2206 // Check the rest of the elements to see if they are consecutive. 2207 for (++i; i != 4; ++i) 2208 if (!isConstantOrUndef(SVOp->getMaskElt(i), ShiftAmt+i)) 2209 return -1; 2210 2211 return ShiftAmt; 2212 } 2213 2214 //===----------------------------------------------------------------------===// 2215 // Addressing Mode Selection 2216 //===----------------------------------------------------------------------===// 2217 2218 /// isIntS16Immediate - This method tests to see if the node is either a 32-bit 2219 /// or 64-bit immediate, and if the value can be accurately represented as a 2220 /// sign extension from a 16-bit value. If so, this returns true and the 2221 /// immediate. 2222 bool llvm::isIntS16Immediate(SDNode *N, int16_t &Imm) { 2223 if (!isa<ConstantSDNode>(N)) 2224 return false; 2225 2226 Imm = (int16_t)cast<ConstantSDNode>(N)->getZExtValue(); 2227 if (N->getValueType(0) == MVT::i32) 2228 return Imm == (int32_t)cast<ConstantSDNode>(N)->getZExtValue(); 2229 else 2230 return Imm == (int64_t)cast<ConstantSDNode>(N)->getZExtValue(); 2231 } 2232 bool llvm::isIntS16Immediate(SDValue Op, int16_t &Imm) { 2233 return isIntS16Immediate(Op.getNode(), Imm); 2234 } 2235 2236 2237 /// SelectAddressEVXRegReg - Given the specified address, check to see if it can 2238 /// be represented as an indexed [r+r] operation. 2239 bool PPCTargetLowering::SelectAddressEVXRegReg(SDValue N, SDValue &Base, 2240 SDValue &Index, 2241 SelectionDAG &DAG) const { 2242 for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); 2243 UI != E; ++UI) { 2244 if (MemSDNode *Memop = dyn_cast<MemSDNode>(*UI)) { 2245 if (Memop->getMemoryVT() == MVT::f64) { 2246 Base = N.getOperand(0); 2247 Index = N.getOperand(1); 2248 return true; 2249 } 2250 } 2251 } 2252 return false; 2253 } 2254 2255 /// SelectAddressRegReg - Given the specified addressed, check to see if it 2256 /// can be represented as an indexed [r+r] operation. Returns false if it 2257 /// can be more efficiently represented as [r+imm]. If \p EncodingAlignment is 2258 /// non-zero and N can be represented by a base register plus a signed 16-bit 2259 /// displacement, make a more precise judgement by checking (displacement % \p 2260 /// EncodingAlignment). 2261 bool PPCTargetLowering::SelectAddressRegReg(SDValue N, SDValue &Base, 2262 SDValue &Index, SelectionDAG &DAG, 2263 unsigned EncodingAlignment) const { 2264 int16_t imm = 0; 2265 if (N.getOpcode() == ISD::ADD) { 2266 // Is there any SPE load/store (f64), which can't handle 16bit offset? 2267 // SPE load/store can only handle 8-bit offsets. 2268 if (hasSPE() && SelectAddressEVXRegReg(N, Base, Index, DAG)) 2269 return true; 2270 if (isIntS16Immediate(N.getOperand(1), imm) && 2271 (!EncodingAlignment || !(imm % EncodingAlignment))) 2272 return false; // r+i 2273 if (N.getOperand(1).getOpcode() == PPCISD::Lo) 2274 return false; // r+i 2275 2276 Base = N.getOperand(0); 2277 Index = N.getOperand(1); 2278 return true; 2279 } else if (N.getOpcode() == ISD::OR) { 2280 if (isIntS16Immediate(N.getOperand(1), imm) && 2281 (!EncodingAlignment || !(imm % EncodingAlignment))) 2282 return false; // r+i can fold it if we can. 2283 2284 // If this is an or of disjoint bitfields, we can codegen this as an add 2285 // (for better address arithmetic) if the LHS and RHS of the OR are provably 2286 // disjoint. 2287 KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0)); 2288 2289 if (LHSKnown.Zero.getBoolValue()) { 2290 KnownBits RHSKnown = DAG.computeKnownBits(N.getOperand(1)); 2291 // If all of the bits are known zero on the LHS or RHS, the add won't 2292 // carry. 2293 if (~(LHSKnown.Zero | RHSKnown.Zero) == 0) { 2294 Base = N.getOperand(0); 2295 Index = N.getOperand(1); 2296 return true; 2297 } 2298 } 2299 } 2300 2301 return false; 2302 } 2303 2304 // If we happen to be doing an i64 load or store into a stack slot that has 2305 // less than a 4-byte alignment, then the frame-index elimination may need to 2306 // use an indexed load or store instruction (because the offset may not be a 2307 // multiple of 4). The extra register needed to hold the offset comes from the 2308 // register scavenger, and it is possible that the scavenger will need to use 2309 // an emergency spill slot. As a result, we need to make sure that a spill slot 2310 // is allocated when doing an i64 load/store into a less-than-4-byte-aligned 2311 // stack slot. 2312 static void fixupFuncForFI(SelectionDAG &DAG, int FrameIdx, EVT VT) { 2313 // FIXME: This does not handle the LWA case. 2314 if (VT != MVT::i64) 2315 return; 2316 2317 // NOTE: We'll exclude negative FIs here, which come from argument 2318 // lowering, because there are no known test cases triggering this problem 2319 // using packed structures (or similar). We can remove this exclusion if 2320 // we find such a test case. The reason why this is so test-case driven is 2321 // because this entire 'fixup' is only to prevent crashes (from the 2322 // register scavenger) on not-really-valid inputs. For example, if we have: 2323 // %a = alloca i1 2324 // %b = bitcast i1* %a to i64* 2325 // store i64* a, i64 b 2326 // then the store should really be marked as 'align 1', but is not. If it 2327 // were marked as 'align 1' then the indexed form would have been 2328 // instruction-selected initially, and the problem this 'fixup' is preventing 2329 // won't happen regardless. 2330 if (FrameIdx < 0) 2331 return; 2332 2333 MachineFunction &MF = DAG.getMachineFunction(); 2334 MachineFrameInfo &MFI = MF.getFrameInfo(); 2335 2336 unsigned Align = MFI.getObjectAlignment(FrameIdx); 2337 if (Align >= 4) 2338 return; 2339 2340 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 2341 FuncInfo->setHasNonRISpills(); 2342 } 2343 2344 /// Returns true if the address N can be represented by a base register plus 2345 /// a signed 16-bit displacement [r+imm], and if it is not better 2346 /// represented as reg+reg. If \p EncodingAlignment is non-zero, only accept 2347 /// displacements that are multiples of that value. 2348 bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp, 2349 SDValue &Base, 2350 SelectionDAG &DAG, 2351 unsigned EncodingAlignment) const { 2352 // FIXME dl should come from parent load or store, not from address 2353 SDLoc dl(N); 2354 // If this can be more profitably realized as r+r, fail. 2355 if (SelectAddressRegReg(N, Disp, Base, DAG, EncodingAlignment)) 2356 return false; 2357 2358 if (N.getOpcode() == ISD::ADD) { 2359 int16_t imm = 0; 2360 if (isIntS16Immediate(N.getOperand(1), imm) && 2361 (!EncodingAlignment || (imm % EncodingAlignment) == 0)) { 2362 Disp = DAG.getTargetConstant(imm, dl, N.getValueType()); 2363 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N.getOperand(0))) { 2364 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); 2365 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); 2366 } else { 2367 Base = N.getOperand(0); 2368 } 2369 return true; // [r+i] 2370 } else if (N.getOperand(1).getOpcode() == PPCISD::Lo) { 2371 // Match LOAD (ADD (X, Lo(G))). 2372 assert(!cast<ConstantSDNode>(N.getOperand(1).getOperand(1))->getZExtValue() 2373 && "Cannot handle constant offsets yet!"); 2374 Disp = N.getOperand(1).getOperand(0); // The global address. 2375 assert(Disp.getOpcode() == ISD::TargetGlobalAddress || 2376 Disp.getOpcode() == ISD::TargetGlobalTLSAddress || 2377 Disp.getOpcode() == ISD::TargetConstantPool || 2378 Disp.getOpcode() == ISD::TargetJumpTable); 2379 Base = N.getOperand(0); 2380 return true; // [&g+r] 2381 } 2382 } else if (N.getOpcode() == ISD::OR) { 2383 int16_t imm = 0; 2384 if (isIntS16Immediate(N.getOperand(1), imm) && 2385 (!EncodingAlignment || (imm % EncodingAlignment) == 0)) { 2386 // If this is an or of disjoint bitfields, we can codegen this as an add 2387 // (for better address arithmetic) if the LHS and RHS of the OR are 2388 // provably disjoint. 2389 KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0)); 2390 2391 if ((LHSKnown.Zero.getZExtValue()|~(uint64_t)imm) == ~0ULL) { 2392 // If all of the bits are known zero on the LHS or RHS, the add won't 2393 // carry. 2394 if (FrameIndexSDNode *FI = 2395 dyn_cast<FrameIndexSDNode>(N.getOperand(0))) { 2396 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); 2397 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); 2398 } else { 2399 Base = N.getOperand(0); 2400 } 2401 Disp = DAG.getTargetConstant(imm, dl, N.getValueType()); 2402 return true; 2403 } 2404 } 2405 } else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) { 2406 // Loading from a constant address. 2407 2408 // If this address fits entirely in a 16-bit sext immediate field, codegen 2409 // this as "d, 0" 2410 int16_t Imm; 2411 if (isIntS16Immediate(CN, Imm) && 2412 (!EncodingAlignment || (Imm % EncodingAlignment) == 0)) { 2413 Disp = DAG.getTargetConstant(Imm, dl, CN->getValueType(0)); 2414 Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO, 2415 CN->getValueType(0)); 2416 return true; 2417 } 2418 2419 // Handle 32-bit sext immediates with LIS + addr mode. 2420 if ((CN->getValueType(0) == MVT::i32 || 2421 (int64_t)CN->getZExtValue() == (int)CN->getZExtValue()) && 2422 (!EncodingAlignment || (CN->getZExtValue() % EncodingAlignment) == 0)) { 2423 int Addr = (int)CN->getZExtValue(); 2424 2425 // Otherwise, break this down into an LIS + disp. 2426 Disp = DAG.getTargetConstant((short)Addr, dl, MVT::i32); 2427 2428 Base = DAG.getTargetConstant((Addr - (signed short)Addr) >> 16, dl, 2429 MVT::i32); 2430 unsigned Opc = CN->getValueType(0) == MVT::i32 ? PPC::LIS : PPC::LIS8; 2431 Base = SDValue(DAG.getMachineNode(Opc, dl, CN->getValueType(0), Base), 0); 2432 return true; 2433 } 2434 } 2435 2436 Disp = DAG.getTargetConstant(0, dl, getPointerTy(DAG.getDataLayout())); 2437 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N)) { 2438 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); 2439 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); 2440 } else 2441 Base = N; 2442 return true; // [r+0] 2443 } 2444 2445 /// SelectAddressRegRegOnly - Given the specified addressed, force it to be 2446 /// represented as an indexed [r+r] operation. 2447 bool PPCTargetLowering::SelectAddressRegRegOnly(SDValue N, SDValue &Base, 2448 SDValue &Index, 2449 SelectionDAG &DAG) const { 2450 // Check to see if we can easily represent this as an [r+r] address. This 2451 // will fail if it thinks that the address is more profitably represented as 2452 // reg+imm, e.g. where imm = 0. 2453 if (SelectAddressRegReg(N, Base, Index, DAG)) 2454 return true; 2455 2456 // If the address is the result of an add, we will utilize the fact that the 2457 // address calculation includes an implicit add. However, we can reduce 2458 // register pressure if we do not materialize a constant just for use as the 2459 // index register. We only get rid of the add if it is not an add of a 2460 // value and a 16-bit signed constant and both have a single use. 2461 int16_t imm = 0; 2462 if (N.getOpcode() == ISD::ADD && 2463 (!isIntS16Immediate(N.getOperand(1), imm) || 2464 !N.getOperand(1).hasOneUse() || !N.getOperand(0).hasOneUse())) { 2465 Base = N.getOperand(0); 2466 Index = N.getOperand(1); 2467 return true; 2468 } 2469 2470 // Otherwise, do it the hard way, using R0 as the base register. 2471 Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO, 2472 N.getValueType()); 2473 Index = N; 2474 return true; 2475 } 2476 2477 /// Returns true if we should use a direct load into vector instruction 2478 /// (such as lxsd or lfd), instead of a load into gpr + direct move sequence. 2479 static bool usePartialVectorLoads(SDNode *N, const PPCSubtarget& ST) { 2480 2481 // If there are any other uses other than scalar to vector, then we should 2482 // keep it as a scalar load -> direct move pattern to prevent multiple 2483 // loads. 2484 LoadSDNode *LD = dyn_cast<LoadSDNode>(N); 2485 if (!LD) 2486 return false; 2487 2488 EVT MemVT = LD->getMemoryVT(); 2489 if (!MemVT.isSimple()) 2490 return false; 2491 switch(MemVT.getSimpleVT().SimpleTy) { 2492 case MVT::i64: 2493 break; 2494 case MVT::i32: 2495 if (!ST.hasP8Vector()) 2496 return false; 2497 break; 2498 case MVT::i16: 2499 case MVT::i8: 2500 if (!ST.hasP9Vector()) 2501 return false; 2502 break; 2503 default: 2504 return false; 2505 } 2506 2507 SDValue LoadedVal(N, 0); 2508 if (!LoadedVal.hasOneUse()) 2509 return false; 2510 2511 for (SDNode::use_iterator UI = LD->use_begin(), UE = LD->use_end(); 2512 UI != UE; ++UI) 2513 if (UI.getUse().get().getResNo() == 0 && 2514 UI->getOpcode() != ISD::SCALAR_TO_VECTOR) 2515 return false; 2516 2517 return true; 2518 } 2519 2520 /// getPreIndexedAddressParts - returns true by value, base pointer and 2521 /// offset pointer and addressing mode by reference if the node's address 2522 /// can be legally represented as pre-indexed load / store address. 2523 bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base, 2524 SDValue &Offset, 2525 ISD::MemIndexedMode &AM, 2526 SelectionDAG &DAG) const { 2527 if (DisablePPCPreinc) return false; 2528 2529 bool isLoad = true; 2530 SDValue Ptr; 2531 EVT VT; 2532 unsigned Alignment; 2533 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) { 2534 Ptr = LD->getBasePtr(); 2535 VT = LD->getMemoryVT(); 2536 Alignment = LD->getAlignment(); 2537 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) { 2538 Ptr = ST->getBasePtr(); 2539 VT = ST->getMemoryVT(); 2540 Alignment = ST->getAlignment(); 2541 isLoad = false; 2542 } else 2543 return false; 2544 2545 // Do not generate pre-inc forms for specific loads that feed scalar_to_vector 2546 // instructions because we can fold these into a more efficient instruction 2547 // instead, (such as LXSD). 2548 if (isLoad && usePartialVectorLoads(N, Subtarget)) { 2549 return false; 2550 } 2551 2552 // PowerPC doesn't have preinc load/store instructions for vectors (except 2553 // for QPX, which does have preinc r+r forms). 2554 if (VT.isVector()) { 2555 if (!Subtarget.hasQPX() || (VT != MVT::v4f64 && VT != MVT::v4f32)) { 2556 return false; 2557 } else if (SelectAddressRegRegOnly(Ptr, Offset, Base, DAG)) { 2558 AM = ISD::PRE_INC; 2559 return true; 2560 } 2561 } 2562 2563 if (SelectAddressRegReg(Ptr, Base, Offset, DAG)) { 2564 // Common code will reject creating a pre-inc form if the base pointer 2565 // is a frame index, or if N is a store and the base pointer is either 2566 // the same as or a predecessor of the value being stored. Check for 2567 // those situations here, and try with swapped Base/Offset instead. 2568 bool Swap = false; 2569 2570 if (isa<FrameIndexSDNode>(Base) || isa<RegisterSDNode>(Base)) 2571 Swap = true; 2572 else if (!isLoad) { 2573 SDValue Val = cast<StoreSDNode>(N)->getValue(); 2574 if (Val == Base || Base.getNode()->isPredecessorOf(Val.getNode())) 2575 Swap = true; 2576 } 2577 2578 if (Swap) 2579 std::swap(Base, Offset); 2580 2581 AM = ISD::PRE_INC; 2582 return true; 2583 } 2584 2585 // LDU/STU can only handle immediates that are a multiple of 4. 2586 if (VT != MVT::i64) { 2587 if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, 0)) 2588 return false; 2589 } else { 2590 // LDU/STU need an address with at least 4-byte alignment. 2591 if (Alignment < 4) 2592 return false; 2593 2594 if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, 4)) 2595 return false; 2596 } 2597 2598 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) { 2599 // PPC64 doesn't have lwau, but it does have lwaux. Reject preinc load of 2600 // sext i32 to i64 when addr mode is r+i. 2601 if (LD->getValueType(0) == MVT::i64 && LD->getMemoryVT() == MVT::i32 && 2602 LD->getExtensionType() == ISD::SEXTLOAD && 2603 isa<ConstantSDNode>(Offset)) 2604 return false; 2605 } 2606 2607 AM = ISD::PRE_INC; 2608 return true; 2609 } 2610 2611 //===----------------------------------------------------------------------===// 2612 // LowerOperation implementation 2613 //===----------------------------------------------------------------------===// 2614 2615 /// Return true if we should reference labels using a PICBase, set the HiOpFlags 2616 /// and LoOpFlags to the target MO flags. 2617 static void getLabelAccessInfo(bool IsPIC, const PPCSubtarget &Subtarget, 2618 unsigned &HiOpFlags, unsigned &LoOpFlags, 2619 const GlobalValue *GV = nullptr) { 2620 HiOpFlags = PPCII::MO_HA; 2621 LoOpFlags = PPCII::MO_LO; 2622 2623 // Don't use the pic base if not in PIC relocation model. 2624 if (IsPIC) { 2625 HiOpFlags |= PPCII::MO_PIC_FLAG; 2626 LoOpFlags |= PPCII::MO_PIC_FLAG; 2627 } 2628 2629 // If this is a reference to a global value that requires a non-lazy-ptr, make 2630 // sure that instruction lowering adds it. 2631 if (GV && Subtarget.hasLazyResolverStub(GV)) { 2632 HiOpFlags |= PPCII::MO_NLP_FLAG; 2633 LoOpFlags |= PPCII::MO_NLP_FLAG; 2634 2635 if (GV->hasHiddenVisibility()) { 2636 HiOpFlags |= PPCII::MO_NLP_HIDDEN_FLAG; 2637 LoOpFlags |= PPCII::MO_NLP_HIDDEN_FLAG; 2638 } 2639 } 2640 } 2641 2642 static SDValue LowerLabelRef(SDValue HiPart, SDValue LoPart, bool isPIC, 2643 SelectionDAG &DAG) { 2644 SDLoc DL(HiPart); 2645 EVT PtrVT = HiPart.getValueType(); 2646 SDValue Zero = DAG.getConstant(0, DL, PtrVT); 2647 2648 SDValue Hi = DAG.getNode(PPCISD::Hi, DL, PtrVT, HiPart, Zero); 2649 SDValue Lo = DAG.getNode(PPCISD::Lo, DL, PtrVT, LoPart, Zero); 2650 2651 // With PIC, the first instruction is actually "GR+hi(&G)". 2652 if (isPIC) 2653 Hi = DAG.getNode(ISD::ADD, DL, PtrVT, 2654 DAG.getNode(PPCISD::GlobalBaseReg, DL, PtrVT), Hi); 2655 2656 // Generate non-pic code that has direct accesses to the constant pool. 2657 // The address of the global is just (hi(&g)+lo(&g)). 2658 return DAG.getNode(ISD::ADD, DL, PtrVT, Hi, Lo); 2659 } 2660 2661 static void setUsesTOCBasePtr(MachineFunction &MF) { 2662 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 2663 FuncInfo->setUsesTOCBasePtr(); 2664 } 2665 2666 static void setUsesTOCBasePtr(SelectionDAG &DAG) { 2667 setUsesTOCBasePtr(DAG.getMachineFunction()); 2668 } 2669 2670 static SDValue getTOCEntry(SelectionDAG &DAG, const SDLoc &dl, bool Is64Bit, 2671 SDValue GA) { 2672 EVT VT = Is64Bit ? MVT::i64 : MVT::i32; 2673 SDValue Reg = Is64Bit ? DAG.getRegister(PPC::X2, VT) : 2674 DAG.getNode(PPCISD::GlobalBaseReg, dl, VT); 2675 2676 SDValue Ops[] = { GA, Reg }; 2677 return DAG.getMemIntrinsicNode( 2678 PPCISD::TOC_ENTRY, dl, DAG.getVTList(VT, MVT::Other), Ops, VT, 2679 MachinePointerInfo::getGOT(DAG.getMachineFunction()), 0, 2680 MachineMemOperand::MOLoad); 2681 } 2682 2683 SDValue PPCTargetLowering::LowerConstantPool(SDValue Op, 2684 SelectionDAG &DAG) const { 2685 EVT PtrVT = Op.getValueType(); 2686 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op); 2687 const Constant *C = CP->getConstVal(); 2688 2689 // 64-bit SVR4 ABI code is always position-independent. 2690 // The actual address of the GlobalValue is stored in the TOC. 2691 if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) { 2692 setUsesTOCBasePtr(DAG); 2693 SDValue GA = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0); 2694 return getTOCEntry(DAG, SDLoc(CP), true, GA); 2695 } 2696 2697 unsigned MOHiFlag, MOLoFlag; 2698 bool IsPIC = isPositionIndependent(); 2699 getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag); 2700 2701 if (IsPIC && Subtarget.isSVR4ABI()) { 2702 SDValue GA = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 2703 PPCII::MO_PIC_FLAG); 2704 return getTOCEntry(DAG, SDLoc(CP), false, GA); 2705 } 2706 2707 SDValue CPIHi = 2708 DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0, MOHiFlag); 2709 SDValue CPILo = 2710 DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0, MOLoFlag); 2711 return LowerLabelRef(CPIHi, CPILo, IsPIC, DAG); 2712 } 2713 2714 // For 64-bit PowerPC, prefer the more compact relative encodings. 2715 // This trades 32 bits per jump table entry for one or two instructions 2716 // on the jump site. 2717 unsigned PPCTargetLowering::getJumpTableEncoding() const { 2718 if (isJumpTableRelative()) 2719 return MachineJumpTableInfo::EK_LabelDifference32; 2720 2721 return TargetLowering::getJumpTableEncoding(); 2722 } 2723 2724 bool PPCTargetLowering::isJumpTableRelative() const { 2725 if (Subtarget.isPPC64()) 2726 return true; 2727 return TargetLowering::isJumpTableRelative(); 2728 } 2729 2730 SDValue PPCTargetLowering::getPICJumpTableRelocBase(SDValue Table, 2731 SelectionDAG &DAG) const { 2732 if (!Subtarget.isPPC64()) 2733 return TargetLowering::getPICJumpTableRelocBase(Table, DAG); 2734 2735 switch (getTargetMachine().getCodeModel()) { 2736 case CodeModel::Small: 2737 case CodeModel::Medium: 2738 return TargetLowering::getPICJumpTableRelocBase(Table, DAG); 2739 default: 2740 return DAG.getNode(PPCISD::GlobalBaseReg, SDLoc(), 2741 getPointerTy(DAG.getDataLayout())); 2742 } 2743 } 2744 2745 const MCExpr * 2746 PPCTargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF, 2747 unsigned JTI, 2748 MCContext &Ctx) const { 2749 if (!Subtarget.isPPC64()) 2750 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx); 2751 2752 switch (getTargetMachine().getCodeModel()) { 2753 case CodeModel::Small: 2754 case CodeModel::Medium: 2755 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx); 2756 default: 2757 return MCSymbolRefExpr::create(MF->getPICBaseSymbol(), Ctx); 2758 } 2759 } 2760 2761 SDValue PPCTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const { 2762 EVT PtrVT = Op.getValueType(); 2763 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op); 2764 2765 // 64-bit SVR4 ABI code is always position-independent. 2766 // The actual address of the GlobalValue is stored in the TOC. 2767 if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) { 2768 setUsesTOCBasePtr(DAG); 2769 SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT); 2770 return getTOCEntry(DAG, SDLoc(JT), true, GA); 2771 } 2772 2773 unsigned MOHiFlag, MOLoFlag; 2774 bool IsPIC = isPositionIndependent(); 2775 getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag); 2776 2777 if (IsPIC && Subtarget.isSVR4ABI()) { 2778 SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, 2779 PPCII::MO_PIC_FLAG); 2780 return getTOCEntry(DAG, SDLoc(GA), false, GA); 2781 } 2782 2783 SDValue JTIHi = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOHiFlag); 2784 SDValue JTILo = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOLoFlag); 2785 return LowerLabelRef(JTIHi, JTILo, IsPIC, DAG); 2786 } 2787 2788 SDValue PPCTargetLowering::LowerBlockAddress(SDValue Op, 2789 SelectionDAG &DAG) const { 2790 EVT PtrVT = Op.getValueType(); 2791 BlockAddressSDNode *BASDN = cast<BlockAddressSDNode>(Op); 2792 const BlockAddress *BA = BASDN->getBlockAddress(); 2793 2794 // 64-bit SVR4 ABI code is always position-independent. 2795 // The actual BlockAddress is stored in the TOC. 2796 if (Subtarget.isSVR4ABI() && 2797 (Subtarget.isPPC64() || isPositionIndependent())) { 2798 if (Subtarget.isPPC64()) 2799 setUsesTOCBasePtr(DAG); 2800 SDValue GA = DAG.getTargetBlockAddress(BA, PtrVT, BASDN->getOffset()); 2801 return getTOCEntry(DAG, SDLoc(BASDN), Subtarget.isPPC64(), GA); 2802 } 2803 2804 unsigned MOHiFlag, MOLoFlag; 2805 bool IsPIC = isPositionIndependent(); 2806 getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag); 2807 SDValue TgtBAHi = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOHiFlag); 2808 SDValue TgtBALo = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOLoFlag); 2809 return LowerLabelRef(TgtBAHi, TgtBALo, IsPIC, DAG); 2810 } 2811 2812 SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op, 2813 SelectionDAG &DAG) const { 2814 // FIXME: TLS addresses currently use medium model code sequences, 2815 // which is the most useful form. Eventually support for small and 2816 // large models could be added if users need it, at the cost of 2817 // additional complexity. 2818 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); 2819 if (DAG.getTarget().useEmulatedTLS()) 2820 return LowerToTLSEmulatedModel(GA, DAG); 2821 2822 SDLoc dl(GA); 2823 const GlobalValue *GV = GA->getGlobal(); 2824 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2825 bool is64bit = Subtarget.isPPC64(); 2826 const Module *M = DAG.getMachineFunction().getFunction().getParent(); 2827 PICLevel::Level picLevel = M->getPICLevel(); 2828 2829 const TargetMachine &TM = getTargetMachine(); 2830 TLSModel::Model Model = TM.getTLSModel(GV); 2831 2832 if (Model == TLSModel::LocalExec) { 2833 SDValue TGAHi = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 2834 PPCII::MO_TPREL_HA); 2835 SDValue TGALo = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 2836 PPCII::MO_TPREL_LO); 2837 SDValue TLSReg = is64bit ? DAG.getRegister(PPC::X13, MVT::i64) 2838 : DAG.getRegister(PPC::R2, MVT::i32); 2839 2840 SDValue Hi = DAG.getNode(PPCISD::Hi, dl, PtrVT, TGAHi, TLSReg); 2841 return DAG.getNode(PPCISD::Lo, dl, PtrVT, TGALo, Hi); 2842 } 2843 2844 if (Model == TLSModel::InitialExec) { 2845 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); 2846 SDValue TGATLS = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 2847 PPCII::MO_TLS); 2848 SDValue GOTPtr; 2849 if (is64bit) { 2850 setUsesTOCBasePtr(DAG); 2851 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); 2852 GOTPtr = DAG.getNode(PPCISD::ADDIS_GOT_TPREL_HA, dl, 2853 PtrVT, GOTReg, TGA); 2854 } else { 2855 if (!TM.isPositionIndependent()) 2856 GOTPtr = DAG.getNode(PPCISD::PPC32_GOT, dl, PtrVT); 2857 else if (picLevel == PICLevel::SmallPIC) 2858 GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT); 2859 else 2860 GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT); 2861 } 2862 SDValue TPOffset = DAG.getNode(PPCISD::LD_GOT_TPREL_L, dl, 2863 PtrVT, TGA, GOTPtr); 2864 return DAG.getNode(PPCISD::ADD_TLS, dl, PtrVT, TPOffset, TGATLS); 2865 } 2866 2867 if (Model == TLSModel::GeneralDynamic) { 2868 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); 2869 SDValue GOTPtr; 2870 if (is64bit) { 2871 setUsesTOCBasePtr(DAG); 2872 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); 2873 GOTPtr = DAG.getNode(PPCISD::ADDIS_TLSGD_HA, dl, PtrVT, 2874 GOTReg, TGA); 2875 } else { 2876 if (picLevel == PICLevel::SmallPIC) 2877 GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT); 2878 else 2879 GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT); 2880 } 2881 return DAG.getNode(PPCISD::ADDI_TLSGD_L_ADDR, dl, PtrVT, 2882 GOTPtr, TGA, TGA); 2883 } 2884 2885 if (Model == TLSModel::LocalDynamic) { 2886 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); 2887 SDValue GOTPtr; 2888 if (is64bit) { 2889 setUsesTOCBasePtr(DAG); 2890 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); 2891 GOTPtr = DAG.getNode(PPCISD::ADDIS_TLSLD_HA, dl, PtrVT, 2892 GOTReg, TGA); 2893 } else { 2894 if (picLevel == PICLevel::SmallPIC) 2895 GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT); 2896 else 2897 GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT); 2898 } 2899 SDValue TLSAddr = DAG.getNode(PPCISD::ADDI_TLSLD_L_ADDR, dl, 2900 PtrVT, GOTPtr, TGA, TGA); 2901 SDValue DtvOffsetHi = DAG.getNode(PPCISD::ADDIS_DTPREL_HA, dl, 2902 PtrVT, TLSAddr, TGA); 2903 return DAG.getNode(PPCISD::ADDI_DTPREL_L, dl, PtrVT, DtvOffsetHi, TGA); 2904 } 2905 2906 llvm_unreachable("Unknown TLS model!"); 2907 } 2908 2909 SDValue PPCTargetLowering::LowerGlobalAddress(SDValue Op, 2910 SelectionDAG &DAG) const { 2911 EVT PtrVT = Op.getValueType(); 2912 GlobalAddressSDNode *GSDN = cast<GlobalAddressSDNode>(Op); 2913 SDLoc DL(GSDN); 2914 const GlobalValue *GV = GSDN->getGlobal(); 2915 2916 // 64-bit SVR4 ABI code is always position-independent. 2917 // The actual address of the GlobalValue is stored in the TOC. 2918 if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) { 2919 setUsesTOCBasePtr(DAG); 2920 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset()); 2921 return getTOCEntry(DAG, DL, true, GA); 2922 } 2923 2924 unsigned MOHiFlag, MOLoFlag; 2925 bool IsPIC = isPositionIndependent(); 2926 getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag, GV); 2927 2928 if (IsPIC && Subtarget.isSVR4ABI()) { 2929 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 2930 GSDN->getOffset(), 2931 PPCII::MO_PIC_FLAG); 2932 return getTOCEntry(DAG, DL, false, GA); 2933 } 2934 2935 SDValue GAHi = 2936 DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), MOHiFlag); 2937 SDValue GALo = 2938 DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), MOLoFlag); 2939 2940 SDValue Ptr = LowerLabelRef(GAHi, GALo, IsPIC, DAG); 2941 2942 // If the global reference is actually to a non-lazy-pointer, we have to do an 2943 // extra load to get the address of the global. 2944 if (MOHiFlag & PPCII::MO_NLP_FLAG) 2945 Ptr = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo()); 2946 return Ptr; 2947 } 2948 2949 SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { 2950 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); 2951 SDLoc dl(Op); 2952 2953 if (Op.getValueType() == MVT::v2i64) { 2954 // When the operands themselves are v2i64 values, we need to do something 2955 // special because VSX has no underlying comparison operations for these. 2956 if (Op.getOperand(0).getValueType() == MVT::v2i64) { 2957 // Equality can be handled by casting to the legal type for Altivec 2958 // comparisons, everything else needs to be expanded. 2959 if (CC == ISD::SETEQ || CC == ISD::SETNE) { 2960 return DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, 2961 DAG.getSetCC(dl, MVT::v4i32, 2962 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op.getOperand(0)), 2963 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op.getOperand(1)), 2964 CC)); 2965 } 2966 2967 return SDValue(); 2968 } 2969 2970 // We handle most of these in the usual way. 2971 return Op; 2972 } 2973 2974 // If we're comparing for equality to zero, expose the fact that this is 2975 // implemented as a ctlz/srl pair on ppc, so that the dag combiner can 2976 // fold the new nodes. 2977 if (SDValue V = lowerCmpEqZeroToCtlzSrl(Op, DAG)) 2978 return V; 2979 2980 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2981 // Leave comparisons against 0 and -1 alone for now, since they're usually 2982 // optimized. FIXME: revisit this when we can custom lower all setcc 2983 // optimizations. 2984 if (C->isAllOnesValue() || C->isNullValue()) 2985 return SDValue(); 2986 } 2987 2988 // If we have an integer seteq/setne, turn it into a compare against zero 2989 // by xor'ing the rhs with the lhs, which is faster than setting a 2990 // condition register, reading it back out, and masking the correct bit. The 2991 // normal approach here uses sub to do this instead of xor. Using xor exposes 2992 // the result to other bit-twiddling opportunities. 2993 EVT LHSVT = Op.getOperand(0).getValueType(); 2994 if (LHSVT.isInteger() && (CC == ISD::SETEQ || CC == ISD::SETNE)) { 2995 EVT VT = Op.getValueType(); 2996 SDValue Sub = DAG.getNode(ISD::XOR, dl, LHSVT, Op.getOperand(0), 2997 Op.getOperand(1)); 2998 return DAG.getSetCC(dl, VT, Sub, DAG.getConstant(0, dl, LHSVT), CC); 2999 } 3000 return SDValue(); 3001 } 3002 3003 SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const { 3004 SDNode *Node = Op.getNode(); 3005 EVT VT = Node->getValueType(0); 3006 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 3007 SDValue InChain = Node->getOperand(0); 3008 SDValue VAListPtr = Node->getOperand(1); 3009 const Value *SV = cast<SrcValueSDNode>(Node->getOperand(2))->getValue(); 3010 SDLoc dl(Node); 3011 3012 assert(!Subtarget.isPPC64() && "LowerVAARG is PPC32 only"); 3013 3014 // gpr_index 3015 SDValue GprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain, 3016 VAListPtr, MachinePointerInfo(SV), MVT::i8); 3017 InChain = GprIndex.getValue(1); 3018 3019 if (VT == MVT::i64) { 3020 // Check if GprIndex is even 3021 SDValue GprAnd = DAG.getNode(ISD::AND, dl, MVT::i32, GprIndex, 3022 DAG.getConstant(1, dl, MVT::i32)); 3023 SDValue CC64 = DAG.getSetCC(dl, MVT::i32, GprAnd, 3024 DAG.getConstant(0, dl, MVT::i32), ISD::SETNE); 3025 SDValue GprIndexPlusOne = DAG.getNode(ISD::ADD, dl, MVT::i32, GprIndex, 3026 DAG.getConstant(1, dl, MVT::i32)); 3027 // Align GprIndex to be even if it isn't 3028 GprIndex = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC64, GprIndexPlusOne, 3029 GprIndex); 3030 } 3031 3032 // fpr index is 1 byte after gpr 3033 SDValue FprPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr, 3034 DAG.getConstant(1, dl, MVT::i32)); 3035 3036 // fpr 3037 SDValue FprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain, 3038 FprPtr, MachinePointerInfo(SV), MVT::i8); 3039 InChain = FprIndex.getValue(1); 3040 3041 SDValue RegSaveAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr, 3042 DAG.getConstant(8, dl, MVT::i32)); 3043 3044 SDValue OverflowAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr, 3045 DAG.getConstant(4, dl, MVT::i32)); 3046 3047 // areas 3048 SDValue OverflowArea = 3049 DAG.getLoad(MVT::i32, dl, InChain, OverflowAreaPtr, MachinePointerInfo()); 3050 InChain = OverflowArea.getValue(1); 3051 3052 SDValue RegSaveArea = 3053 DAG.getLoad(MVT::i32, dl, InChain, RegSaveAreaPtr, MachinePointerInfo()); 3054 InChain = RegSaveArea.getValue(1); 3055 3056 // select overflow_area if index > 8 3057 SDValue CC = DAG.getSetCC(dl, MVT::i32, VT.isInteger() ? GprIndex : FprIndex, 3058 DAG.getConstant(8, dl, MVT::i32), ISD::SETLT); 3059 3060 // adjustment constant gpr_index * 4/8 3061 SDValue RegConstant = DAG.getNode(ISD::MUL, dl, MVT::i32, 3062 VT.isInteger() ? GprIndex : FprIndex, 3063 DAG.getConstant(VT.isInteger() ? 4 : 8, dl, 3064 MVT::i32)); 3065 3066 // OurReg = RegSaveArea + RegConstant 3067 SDValue OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, RegSaveArea, 3068 RegConstant); 3069 3070 // Floating types are 32 bytes into RegSaveArea 3071 if (VT.isFloatingPoint()) 3072 OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, OurReg, 3073 DAG.getConstant(32, dl, MVT::i32)); 3074 3075 // increase {f,g}pr_index by 1 (or 2 if VT is i64) 3076 SDValue IndexPlus1 = DAG.getNode(ISD::ADD, dl, MVT::i32, 3077 VT.isInteger() ? GprIndex : FprIndex, 3078 DAG.getConstant(VT == MVT::i64 ? 2 : 1, dl, 3079 MVT::i32)); 3080 3081 InChain = DAG.getTruncStore(InChain, dl, IndexPlus1, 3082 VT.isInteger() ? VAListPtr : FprPtr, 3083 MachinePointerInfo(SV), MVT::i8); 3084 3085 // determine if we should load from reg_save_area or overflow_area 3086 SDValue Result = DAG.getNode(ISD::SELECT, dl, PtrVT, CC, OurReg, OverflowArea); 3087 3088 // increase overflow_area by 4/8 if gpr/fpr > 8 3089 SDValue OverflowAreaPlusN = DAG.getNode(ISD::ADD, dl, PtrVT, OverflowArea, 3090 DAG.getConstant(VT.isInteger() ? 4 : 8, 3091 dl, MVT::i32)); 3092 3093 OverflowArea = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC, OverflowArea, 3094 OverflowAreaPlusN); 3095 3096 InChain = DAG.getTruncStore(InChain, dl, OverflowArea, OverflowAreaPtr, 3097 MachinePointerInfo(), MVT::i32); 3098 3099 return DAG.getLoad(VT, dl, InChain, Result, MachinePointerInfo()); 3100 } 3101 3102 SDValue PPCTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const { 3103 assert(!Subtarget.isPPC64() && "LowerVACOPY is PPC32 only"); 3104 3105 // We have to copy the entire va_list struct: 3106 // 2*sizeof(char) + 2 Byte alignment + 2*sizeof(char*) = 12 Byte 3107 return DAG.getMemcpy(Op.getOperand(0), Op, 3108 Op.getOperand(1), Op.getOperand(2), 3109 DAG.getConstant(12, SDLoc(Op), MVT::i32), 8, false, true, 3110 false, MachinePointerInfo(), MachinePointerInfo()); 3111 } 3112 3113 SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op, 3114 SelectionDAG &DAG) const { 3115 return Op.getOperand(0); 3116 } 3117 3118 SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op, 3119 SelectionDAG &DAG) const { 3120 SDValue Chain = Op.getOperand(0); 3121 SDValue Trmp = Op.getOperand(1); // trampoline 3122 SDValue FPtr = Op.getOperand(2); // nested function 3123 SDValue Nest = Op.getOperand(3); // 'nest' parameter value 3124 SDLoc dl(Op); 3125 3126 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 3127 bool isPPC64 = (PtrVT == MVT::i64); 3128 Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext()); 3129 3130 TargetLowering::ArgListTy Args; 3131 TargetLowering::ArgListEntry Entry; 3132 3133 Entry.Ty = IntPtrTy; 3134 Entry.Node = Trmp; Args.push_back(Entry); 3135 3136 // TrampSize == (isPPC64 ? 48 : 40); 3137 Entry.Node = DAG.getConstant(isPPC64 ? 48 : 40, dl, 3138 isPPC64 ? MVT::i64 : MVT::i32); 3139 Args.push_back(Entry); 3140 3141 Entry.Node = FPtr; Args.push_back(Entry); 3142 Entry.Node = Nest; Args.push_back(Entry); 3143 3144 // Lower to a call to __trampoline_setup(Trmp, TrampSize, FPtr, ctx_reg) 3145 TargetLowering::CallLoweringInfo CLI(DAG); 3146 CLI.setDebugLoc(dl).setChain(Chain).setLibCallee( 3147 CallingConv::C, Type::getVoidTy(*DAG.getContext()), 3148 DAG.getExternalSymbol("__trampoline_setup", PtrVT), std::move(Args)); 3149 3150 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI); 3151 return CallResult.second; 3152 } 3153 3154 SDValue PPCTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const { 3155 MachineFunction &MF = DAG.getMachineFunction(); 3156 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 3157 EVT PtrVT = getPointerTy(MF.getDataLayout()); 3158 3159 SDLoc dl(Op); 3160 3161 if (Subtarget.isDarwinABI() || Subtarget.isPPC64()) { 3162 // vastart just stores the address of the VarArgsFrameIndex slot into the 3163 // memory location argument. 3164 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 3165 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 3166 return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1), 3167 MachinePointerInfo(SV)); 3168 } 3169 3170 // For the 32-bit SVR4 ABI we follow the layout of the va_list struct. 3171 // We suppose the given va_list is already allocated. 3172 // 3173 // typedef struct { 3174 // char gpr; /* index into the array of 8 GPRs 3175 // * stored in the register save area 3176 // * gpr=0 corresponds to r3, 3177 // * gpr=1 to r4, etc. 3178 // */ 3179 // char fpr; /* index into the array of 8 FPRs 3180 // * stored in the register save area 3181 // * fpr=0 corresponds to f1, 3182 // * fpr=1 to f2, etc. 3183 // */ 3184 // char *overflow_arg_area; 3185 // /* location on stack that holds 3186 // * the next overflow argument 3187 // */ 3188 // char *reg_save_area; 3189 // /* where r3:r10 and f1:f8 (if saved) 3190 // * are stored 3191 // */ 3192 // } va_list[1]; 3193 3194 SDValue ArgGPR = DAG.getConstant(FuncInfo->getVarArgsNumGPR(), dl, MVT::i32); 3195 SDValue ArgFPR = DAG.getConstant(FuncInfo->getVarArgsNumFPR(), dl, MVT::i32); 3196 SDValue StackOffsetFI = DAG.getFrameIndex(FuncInfo->getVarArgsStackOffset(), 3197 PtrVT); 3198 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), 3199 PtrVT); 3200 3201 uint64_t FrameOffset = PtrVT.getSizeInBits()/8; 3202 SDValue ConstFrameOffset = DAG.getConstant(FrameOffset, dl, PtrVT); 3203 3204 uint64_t StackOffset = PtrVT.getSizeInBits()/8 - 1; 3205 SDValue ConstStackOffset = DAG.getConstant(StackOffset, dl, PtrVT); 3206 3207 uint64_t FPROffset = 1; 3208 SDValue ConstFPROffset = DAG.getConstant(FPROffset, dl, PtrVT); 3209 3210 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 3211 3212 // Store first byte : number of int regs 3213 SDValue firstStore = 3214 DAG.getTruncStore(Op.getOperand(0), dl, ArgGPR, Op.getOperand(1), 3215 MachinePointerInfo(SV), MVT::i8); 3216 uint64_t nextOffset = FPROffset; 3217 SDValue nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, Op.getOperand(1), 3218 ConstFPROffset); 3219 3220 // Store second byte : number of float regs 3221 SDValue secondStore = 3222 DAG.getTruncStore(firstStore, dl, ArgFPR, nextPtr, 3223 MachinePointerInfo(SV, nextOffset), MVT::i8); 3224 nextOffset += StackOffset; 3225 nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, nextPtr, ConstStackOffset); 3226 3227 // Store second word : arguments given on stack 3228 SDValue thirdStore = DAG.getStore(secondStore, dl, StackOffsetFI, nextPtr, 3229 MachinePointerInfo(SV, nextOffset)); 3230 nextOffset += FrameOffset; 3231 nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, nextPtr, ConstFrameOffset); 3232 3233 // Store third word : arguments given in registers 3234 return DAG.getStore(thirdStore, dl, FR, nextPtr, 3235 MachinePointerInfo(SV, nextOffset)); 3236 } 3237 3238 /// FPR - The set of FP registers that should be allocated for arguments, 3239 /// on Darwin. 3240 static const MCPhysReg FPR[] = {PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, 3241 PPC::F6, PPC::F7, PPC::F8, PPC::F9, PPC::F10, 3242 PPC::F11, PPC::F12, PPC::F13}; 3243 3244 /// QFPR - The set of QPX registers that should be allocated for arguments. 3245 static const MCPhysReg QFPR[] = { 3246 PPC::QF1, PPC::QF2, PPC::QF3, PPC::QF4, PPC::QF5, PPC::QF6, PPC::QF7, 3247 PPC::QF8, PPC::QF9, PPC::QF10, PPC::QF11, PPC::QF12, PPC::QF13}; 3248 3249 /// CalculateStackSlotSize - Calculates the size reserved for this argument on 3250 /// the stack. 3251 static unsigned CalculateStackSlotSize(EVT ArgVT, ISD::ArgFlagsTy Flags, 3252 unsigned PtrByteSize) { 3253 unsigned ArgSize = ArgVT.getStoreSize(); 3254 if (Flags.isByVal()) 3255 ArgSize = Flags.getByValSize(); 3256 3257 // Round up to multiples of the pointer size, except for array members, 3258 // which are always packed. 3259 if (!Flags.isInConsecutiveRegs()) 3260 ArgSize = ((ArgSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 3261 3262 return ArgSize; 3263 } 3264 3265 /// CalculateStackSlotAlignment - Calculates the alignment of this argument 3266 /// on the stack. 3267 static unsigned CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT, 3268 ISD::ArgFlagsTy Flags, 3269 unsigned PtrByteSize) { 3270 unsigned Align = PtrByteSize; 3271 3272 // Altivec parameters are padded to a 16 byte boundary. 3273 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 || 3274 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 || 3275 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 || 3276 ArgVT == MVT::v1i128 || ArgVT == MVT::f128) 3277 Align = 16; 3278 // QPX vector types stored in double-precision are padded to a 32 byte 3279 // boundary. 3280 else if (ArgVT == MVT::v4f64 || ArgVT == MVT::v4i1) 3281 Align = 32; 3282 3283 // ByVal parameters are aligned as requested. 3284 if (Flags.isByVal()) { 3285 unsigned BVAlign = Flags.getByValAlign(); 3286 if (BVAlign > PtrByteSize) { 3287 if (BVAlign % PtrByteSize != 0) 3288 llvm_unreachable( 3289 "ByVal alignment is not a multiple of the pointer size"); 3290 3291 Align = BVAlign; 3292 } 3293 } 3294 3295 // Array members are always packed to their original alignment. 3296 if (Flags.isInConsecutiveRegs()) { 3297 // If the array member was split into multiple registers, the first 3298 // needs to be aligned to the size of the full type. (Except for 3299 // ppcf128, which is only aligned as its f64 components.) 3300 if (Flags.isSplit() && OrigVT != MVT::ppcf128) 3301 Align = OrigVT.getStoreSize(); 3302 else 3303 Align = ArgVT.getStoreSize(); 3304 } 3305 3306 return Align; 3307 } 3308 3309 /// CalculateStackSlotUsed - Return whether this argument will use its 3310 /// stack slot (instead of being passed in registers). ArgOffset, 3311 /// AvailableFPRs, and AvailableVRs must hold the current argument 3312 /// position, and will be updated to account for this argument. 3313 static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT, 3314 ISD::ArgFlagsTy Flags, 3315 unsigned PtrByteSize, 3316 unsigned LinkageSize, 3317 unsigned ParamAreaSize, 3318 unsigned &ArgOffset, 3319 unsigned &AvailableFPRs, 3320 unsigned &AvailableVRs, bool HasQPX) { 3321 bool UseMemory = false; 3322 3323 // Respect alignment of argument on the stack. 3324 unsigned Align = 3325 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize); 3326 ArgOffset = ((ArgOffset + Align - 1) / Align) * Align; 3327 // If there's no space left in the argument save area, we must 3328 // use memory (this check also catches zero-sized arguments). 3329 if (ArgOffset >= LinkageSize + ParamAreaSize) 3330 UseMemory = true; 3331 3332 // Allocate argument on the stack. 3333 ArgOffset += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize); 3334 if (Flags.isInConsecutiveRegsLast()) 3335 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 3336 // If we overran the argument save area, we must use memory 3337 // (this check catches arguments passed partially in memory) 3338 if (ArgOffset > LinkageSize + ParamAreaSize) 3339 UseMemory = true; 3340 3341 // However, if the argument is actually passed in an FPR or a VR, 3342 // we don't use memory after all. 3343 if (!Flags.isByVal()) { 3344 if (ArgVT == MVT::f32 || ArgVT == MVT::f64 || 3345 // QPX registers overlap with the scalar FP registers. 3346 (HasQPX && (ArgVT == MVT::v4f32 || 3347 ArgVT == MVT::v4f64 || 3348 ArgVT == MVT::v4i1))) 3349 if (AvailableFPRs > 0) { 3350 --AvailableFPRs; 3351 return false; 3352 } 3353 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 || 3354 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 || 3355 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 || 3356 ArgVT == MVT::v1i128 || ArgVT == MVT::f128) 3357 if (AvailableVRs > 0) { 3358 --AvailableVRs; 3359 return false; 3360 } 3361 } 3362 3363 return UseMemory; 3364 } 3365 3366 /// EnsureStackAlignment - Round stack frame size up from NumBytes to 3367 /// ensure minimum alignment required for target. 3368 static unsigned EnsureStackAlignment(const PPCFrameLowering *Lowering, 3369 unsigned NumBytes) { 3370 unsigned TargetAlign = Lowering->getStackAlignment(); 3371 unsigned AlignMask = TargetAlign - 1; 3372 NumBytes = (NumBytes + AlignMask) & ~AlignMask; 3373 return NumBytes; 3374 } 3375 3376 SDValue PPCTargetLowering::LowerFormalArguments( 3377 SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 3378 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 3379 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 3380 if (Subtarget.isSVR4ABI()) { 3381 if (Subtarget.isPPC64()) 3382 return LowerFormalArguments_64SVR4(Chain, CallConv, isVarArg, Ins, 3383 dl, DAG, InVals); 3384 else 3385 return LowerFormalArguments_32SVR4(Chain, CallConv, isVarArg, Ins, 3386 dl, DAG, InVals); 3387 } else { 3388 return LowerFormalArguments_Darwin(Chain, CallConv, isVarArg, Ins, 3389 dl, DAG, InVals); 3390 } 3391 } 3392 3393 SDValue PPCTargetLowering::LowerFormalArguments_32SVR4( 3394 SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 3395 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 3396 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 3397 3398 // 32-bit SVR4 ABI Stack Frame Layout: 3399 // +-----------------------------------+ 3400 // +--> | Back chain | 3401 // | +-----------------------------------+ 3402 // | | Floating-point register save area | 3403 // | +-----------------------------------+ 3404 // | | General register save area | 3405 // | +-----------------------------------+ 3406 // | | CR save word | 3407 // | +-----------------------------------+ 3408 // | | VRSAVE save word | 3409 // | +-----------------------------------+ 3410 // | | Alignment padding | 3411 // | +-----------------------------------+ 3412 // | | Vector register save area | 3413 // | +-----------------------------------+ 3414 // | | Local variable space | 3415 // | +-----------------------------------+ 3416 // | | Parameter list area | 3417 // | +-----------------------------------+ 3418 // | | LR save word | 3419 // | +-----------------------------------+ 3420 // SP--> +--- | Back chain | 3421 // +-----------------------------------+ 3422 // 3423 // Specifications: 3424 // System V Application Binary Interface PowerPC Processor Supplement 3425 // AltiVec Technology Programming Interface Manual 3426 3427 MachineFunction &MF = DAG.getMachineFunction(); 3428 MachineFrameInfo &MFI = MF.getFrameInfo(); 3429 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 3430 3431 EVT PtrVT = getPointerTy(MF.getDataLayout()); 3432 // Potential tail calls could cause overwriting of argument stack slots. 3433 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && 3434 (CallConv == CallingConv::Fast)); 3435 unsigned PtrByteSize = 4; 3436 3437 // Assign locations to all of the incoming arguments. 3438 SmallVector<CCValAssign, 16> ArgLocs; 3439 PPCCCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs, 3440 *DAG.getContext()); 3441 3442 // Reserve space for the linkage area on the stack. 3443 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 3444 CCInfo.AllocateStack(LinkageSize, PtrByteSize); 3445 if (useSoftFloat()) 3446 CCInfo.PreAnalyzeFormalArguments(Ins); 3447 3448 CCInfo.AnalyzeFormalArguments(Ins, CC_PPC32_SVR4); 3449 CCInfo.clearWasPPCF128(); 3450 3451 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 3452 CCValAssign &VA = ArgLocs[i]; 3453 3454 // Arguments stored in registers. 3455 if (VA.isRegLoc()) { 3456 const TargetRegisterClass *RC; 3457 EVT ValVT = VA.getValVT(); 3458 3459 switch (ValVT.getSimpleVT().SimpleTy) { 3460 default: 3461 llvm_unreachable("ValVT not supported by formal arguments Lowering"); 3462 case MVT::i1: 3463 case MVT::i32: 3464 RC = &PPC::GPRCRegClass; 3465 break; 3466 case MVT::f32: 3467 if (Subtarget.hasP8Vector()) 3468 RC = &PPC::VSSRCRegClass; 3469 else if (Subtarget.hasSPE()) 3470 RC = &PPC::SPE4RCRegClass; 3471 else 3472 RC = &PPC::F4RCRegClass; 3473 break; 3474 case MVT::f64: 3475 if (Subtarget.hasVSX()) 3476 RC = &PPC::VSFRCRegClass; 3477 else if (Subtarget.hasSPE()) 3478 // SPE passes doubles in GPR pairs. 3479 RC = &PPC::GPRCRegClass; 3480 else 3481 RC = &PPC::F8RCRegClass; 3482 break; 3483 case MVT::v16i8: 3484 case MVT::v8i16: 3485 case MVT::v4i32: 3486 RC = &PPC::VRRCRegClass; 3487 break; 3488 case MVT::v4f32: 3489 RC = Subtarget.hasQPX() ? &PPC::QSRCRegClass : &PPC::VRRCRegClass; 3490 break; 3491 case MVT::v2f64: 3492 case MVT::v2i64: 3493 RC = &PPC::VRRCRegClass; 3494 break; 3495 case MVT::v4f64: 3496 RC = &PPC::QFRCRegClass; 3497 break; 3498 case MVT::v4i1: 3499 RC = &PPC::QBRCRegClass; 3500 break; 3501 } 3502 3503 SDValue ArgValue; 3504 // Transform the arguments stored in physical registers into 3505 // virtual ones. 3506 if (VA.getLocVT() == MVT::f64 && Subtarget.hasSPE()) { 3507 assert(i + 1 < e && "No second half of double precision argument"); 3508 unsigned RegLo = MF.addLiveIn(VA.getLocReg(), RC); 3509 unsigned RegHi = MF.addLiveIn(ArgLocs[++i].getLocReg(), RC); 3510 SDValue ArgValueLo = DAG.getCopyFromReg(Chain, dl, RegLo, MVT::i32); 3511 SDValue ArgValueHi = DAG.getCopyFromReg(Chain, dl, RegHi, MVT::i32); 3512 if (!Subtarget.isLittleEndian()) 3513 std::swap (ArgValueLo, ArgValueHi); 3514 ArgValue = DAG.getNode(PPCISD::BUILD_SPE64, dl, MVT::f64, ArgValueLo, 3515 ArgValueHi); 3516 } else { 3517 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC); 3518 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, 3519 ValVT == MVT::i1 ? MVT::i32 : ValVT); 3520 if (ValVT == MVT::i1) 3521 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, ArgValue); 3522 } 3523 3524 InVals.push_back(ArgValue); 3525 } else { 3526 // Argument stored in memory. 3527 assert(VA.isMemLoc()); 3528 3529 // Get the extended size of the argument type in stack 3530 unsigned ArgSize = VA.getLocVT().getStoreSize(); 3531 // Get the actual size of the argument type 3532 unsigned ObjSize = VA.getValVT().getStoreSize(); 3533 unsigned ArgOffset = VA.getLocMemOffset(); 3534 // Stack objects in PPC32 are right justified. 3535 ArgOffset += ArgSize - ObjSize; 3536 int FI = MFI.CreateFixedObject(ArgSize, ArgOffset, isImmutable); 3537 3538 // Create load nodes to retrieve arguments from the stack. 3539 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 3540 InVals.push_back( 3541 DAG.getLoad(VA.getValVT(), dl, Chain, FIN, MachinePointerInfo())); 3542 } 3543 } 3544 3545 // Assign locations to all of the incoming aggregate by value arguments. 3546 // Aggregates passed by value are stored in the local variable space of the 3547 // caller's stack frame, right above the parameter list area. 3548 SmallVector<CCValAssign, 16> ByValArgLocs; 3549 CCState CCByValInfo(CallConv, isVarArg, DAG.getMachineFunction(), 3550 ByValArgLocs, *DAG.getContext()); 3551 3552 // Reserve stack space for the allocations in CCInfo. 3553 CCByValInfo.AllocateStack(CCInfo.getNextStackOffset(), PtrByteSize); 3554 3555 CCByValInfo.AnalyzeFormalArguments(Ins, CC_PPC32_SVR4_ByVal); 3556 3557 // Area that is at least reserved in the caller of this function. 3558 unsigned MinReservedArea = CCByValInfo.getNextStackOffset(); 3559 MinReservedArea = std::max(MinReservedArea, LinkageSize); 3560 3561 // Set the size that is at least reserved in caller of this function. Tail 3562 // call optimized function's reserved stack space needs to be aligned so that 3563 // taking the difference between two stack areas will result in an aligned 3564 // stack. 3565 MinReservedArea = 3566 EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea); 3567 FuncInfo->setMinReservedArea(MinReservedArea); 3568 3569 SmallVector<SDValue, 8> MemOps; 3570 3571 // If the function takes variable number of arguments, make a frame index for 3572 // the start of the first vararg value... for expansion of llvm.va_start. 3573 if (isVarArg) { 3574 static const MCPhysReg GPArgRegs[] = { 3575 PPC::R3, PPC::R4, PPC::R5, PPC::R6, 3576 PPC::R7, PPC::R8, PPC::R9, PPC::R10, 3577 }; 3578 const unsigned NumGPArgRegs = array_lengthof(GPArgRegs); 3579 3580 static const MCPhysReg FPArgRegs[] = { 3581 PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7, 3582 PPC::F8 3583 }; 3584 unsigned NumFPArgRegs = array_lengthof(FPArgRegs); 3585 3586 if (useSoftFloat() || hasSPE()) 3587 NumFPArgRegs = 0; 3588 3589 FuncInfo->setVarArgsNumGPR(CCInfo.getFirstUnallocated(GPArgRegs)); 3590 FuncInfo->setVarArgsNumFPR(CCInfo.getFirstUnallocated(FPArgRegs)); 3591 3592 // Make room for NumGPArgRegs and NumFPArgRegs. 3593 int Depth = NumGPArgRegs * PtrVT.getSizeInBits()/8 + 3594 NumFPArgRegs * MVT(MVT::f64).getSizeInBits()/8; 3595 3596 FuncInfo->setVarArgsStackOffset( 3597 MFI.CreateFixedObject(PtrVT.getSizeInBits()/8, 3598 CCInfo.getNextStackOffset(), true)); 3599 3600 FuncInfo->setVarArgsFrameIndex(MFI.CreateStackObject(Depth, 8, false)); 3601 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 3602 3603 // The fixed integer arguments of a variadic function are stored to the 3604 // VarArgsFrameIndex on the stack so that they may be loaded by 3605 // dereferencing the result of va_next. 3606 for (unsigned GPRIndex = 0; GPRIndex != NumGPArgRegs; ++GPRIndex) { 3607 // Get an existing live-in vreg, or add a new one. 3608 unsigned VReg = MF.getRegInfo().getLiveInVirtReg(GPArgRegs[GPRIndex]); 3609 if (!VReg) 3610 VReg = MF.addLiveIn(GPArgRegs[GPRIndex], &PPC::GPRCRegClass); 3611 3612 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 3613 SDValue Store = 3614 DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo()); 3615 MemOps.push_back(Store); 3616 // Increment the address by four for the next argument to store 3617 SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, dl, PtrVT); 3618 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); 3619 } 3620 3621 // FIXME 32-bit SVR4: We only need to save FP argument registers if CR bit 6 3622 // is set. 3623 // The double arguments are stored to the VarArgsFrameIndex 3624 // on the stack. 3625 for (unsigned FPRIndex = 0; FPRIndex != NumFPArgRegs; ++FPRIndex) { 3626 // Get an existing live-in vreg, or add a new one. 3627 unsigned VReg = MF.getRegInfo().getLiveInVirtReg(FPArgRegs[FPRIndex]); 3628 if (!VReg) 3629 VReg = MF.addLiveIn(FPArgRegs[FPRIndex], &PPC::F8RCRegClass); 3630 3631 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::f64); 3632 SDValue Store = 3633 DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo()); 3634 MemOps.push_back(Store); 3635 // Increment the address by eight for the next argument to store 3636 SDValue PtrOff = DAG.getConstant(MVT(MVT::f64).getSizeInBits()/8, dl, 3637 PtrVT); 3638 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); 3639 } 3640 } 3641 3642 if (!MemOps.empty()) 3643 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); 3644 3645 return Chain; 3646 } 3647 3648 // PPC64 passes i8, i16, and i32 values in i64 registers. Promote 3649 // value to MVT::i64 and then truncate to the correct register size. 3650 SDValue PPCTargetLowering::extendArgForPPC64(ISD::ArgFlagsTy Flags, 3651 EVT ObjectVT, SelectionDAG &DAG, 3652 SDValue ArgVal, 3653 const SDLoc &dl) const { 3654 if (Flags.isSExt()) 3655 ArgVal = DAG.getNode(ISD::AssertSext, dl, MVT::i64, ArgVal, 3656 DAG.getValueType(ObjectVT)); 3657 else if (Flags.isZExt()) 3658 ArgVal = DAG.getNode(ISD::AssertZext, dl, MVT::i64, ArgVal, 3659 DAG.getValueType(ObjectVT)); 3660 3661 return DAG.getNode(ISD::TRUNCATE, dl, ObjectVT, ArgVal); 3662 } 3663 3664 SDValue PPCTargetLowering::LowerFormalArguments_64SVR4( 3665 SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 3666 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 3667 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 3668 // TODO: add description of PPC stack frame format, or at least some docs. 3669 // 3670 bool isELFv2ABI = Subtarget.isELFv2ABI(); 3671 bool isLittleEndian = Subtarget.isLittleEndian(); 3672 MachineFunction &MF = DAG.getMachineFunction(); 3673 MachineFrameInfo &MFI = MF.getFrameInfo(); 3674 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 3675 3676 assert(!(CallConv == CallingConv::Fast && isVarArg) && 3677 "fastcc not supported on varargs functions"); 3678 3679 EVT PtrVT = getPointerTy(MF.getDataLayout()); 3680 // Potential tail calls could cause overwriting of argument stack slots. 3681 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && 3682 (CallConv == CallingConv::Fast)); 3683 unsigned PtrByteSize = 8; 3684 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 3685 3686 static const MCPhysReg GPR[] = { 3687 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 3688 PPC::X7, PPC::X8, PPC::X9, PPC::X10, 3689 }; 3690 static const MCPhysReg VR[] = { 3691 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, 3692 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 3693 }; 3694 3695 const unsigned Num_GPR_Regs = array_lengthof(GPR); 3696 const unsigned Num_FPR_Regs = useSoftFloat() ? 0 : 13; 3697 const unsigned Num_VR_Regs = array_lengthof(VR); 3698 const unsigned Num_QFPR_Regs = Num_FPR_Regs; 3699 3700 // Do a first pass over the arguments to determine whether the ABI 3701 // guarantees that our caller has allocated the parameter save area 3702 // on its stack frame. In the ELFv1 ABI, this is always the case; 3703 // in the ELFv2 ABI, it is true if this is a vararg function or if 3704 // any parameter is located in a stack slot. 3705 3706 bool HasParameterArea = !isELFv2ABI || isVarArg; 3707 unsigned ParamAreaSize = Num_GPR_Regs * PtrByteSize; 3708 unsigned NumBytes = LinkageSize; 3709 unsigned AvailableFPRs = Num_FPR_Regs; 3710 unsigned AvailableVRs = Num_VR_Regs; 3711 for (unsigned i = 0, e = Ins.size(); i != e; ++i) { 3712 if (Ins[i].Flags.isNest()) 3713 continue; 3714 3715 if (CalculateStackSlotUsed(Ins[i].VT, Ins[i].ArgVT, Ins[i].Flags, 3716 PtrByteSize, LinkageSize, ParamAreaSize, 3717 NumBytes, AvailableFPRs, AvailableVRs, 3718 Subtarget.hasQPX())) 3719 HasParameterArea = true; 3720 } 3721 3722 // Add DAG nodes to load the arguments or copy them out of registers. On 3723 // entry to a function on PPC, the arguments start after the linkage area, 3724 // although the first ones are often in registers. 3725 3726 unsigned ArgOffset = LinkageSize; 3727 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; 3728 unsigned &QFPR_idx = FPR_idx; 3729 SmallVector<SDValue, 8> MemOps; 3730 Function::const_arg_iterator FuncArg = MF.getFunction().arg_begin(); 3731 unsigned CurArgIdx = 0; 3732 for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) { 3733 SDValue ArgVal; 3734 bool needsLoad = false; 3735 EVT ObjectVT = Ins[ArgNo].VT; 3736 EVT OrigVT = Ins[ArgNo].ArgVT; 3737 unsigned ObjSize = ObjectVT.getStoreSize(); 3738 unsigned ArgSize = ObjSize; 3739 ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags; 3740 if (Ins[ArgNo].isOrigArg()) { 3741 std::advance(FuncArg, Ins[ArgNo].getOrigArgIndex() - CurArgIdx); 3742 CurArgIdx = Ins[ArgNo].getOrigArgIndex(); 3743 } 3744 // We re-align the argument offset for each argument, except when using the 3745 // fast calling convention, when we need to make sure we do that only when 3746 // we'll actually use a stack slot. 3747 unsigned CurArgOffset, Align; 3748 auto ComputeArgOffset = [&]() { 3749 /* Respect alignment of argument on the stack. */ 3750 Align = CalculateStackSlotAlignment(ObjectVT, OrigVT, Flags, PtrByteSize); 3751 ArgOffset = ((ArgOffset + Align - 1) / Align) * Align; 3752 CurArgOffset = ArgOffset; 3753 }; 3754 3755 if (CallConv != CallingConv::Fast) { 3756 ComputeArgOffset(); 3757 3758 /* Compute GPR index associated with argument offset. */ 3759 GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize; 3760 GPR_idx = std::min(GPR_idx, Num_GPR_Regs); 3761 } 3762 3763 // FIXME the codegen can be much improved in some cases. 3764 // We do not have to keep everything in memory. 3765 if (Flags.isByVal()) { 3766 assert(Ins[ArgNo].isOrigArg() && "Byval arguments cannot be implicit"); 3767 3768 if (CallConv == CallingConv::Fast) 3769 ComputeArgOffset(); 3770 3771 // ObjSize is the true size, ArgSize rounded up to multiple of registers. 3772 ObjSize = Flags.getByValSize(); 3773 ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 3774 // Empty aggregate parameters do not take up registers. Examples: 3775 // struct { } a; 3776 // union { } b; 3777 // int c[0]; 3778 // etc. However, we have to provide a place-holder in InVals, so 3779 // pretend we have an 8-byte item at the current address for that 3780 // purpose. 3781 if (!ObjSize) { 3782 int FI = MFI.CreateFixedObject(PtrByteSize, ArgOffset, true); 3783 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 3784 InVals.push_back(FIN); 3785 continue; 3786 } 3787 3788 // Create a stack object covering all stack doublewords occupied 3789 // by the argument. If the argument is (fully or partially) on 3790 // the stack, or if the argument is fully in registers but the 3791 // caller has allocated the parameter save anyway, we can refer 3792 // directly to the caller's stack frame. Otherwise, create a 3793 // local copy in our own frame. 3794 int FI; 3795 if (HasParameterArea || 3796 ArgSize + ArgOffset > LinkageSize + Num_GPR_Regs * PtrByteSize) 3797 FI = MFI.CreateFixedObject(ArgSize, ArgOffset, false, true); 3798 else 3799 FI = MFI.CreateStackObject(ArgSize, Align, false); 3800 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 3801 3802 // Handle aggregates smaller than 8 bytes. 3803 if (ObjSize < PtrByteSize) { 3804 // The value of the object is its address, which differs from the 3805 // address of the enclosing doubleword on big-endian systems. 3806 SDValue Arg = FIN; 3807 if (!isLittleEndian) { 3808 SDValue ArgOff = DAG.getConstant(PtrByteSize - ObjSize, dl, PtrVT); 3809 Arg = DAG.getNode(ISD::ADD, dl, ArgOff.getValueType(), Arg, ArgOff); 3810 } 3811 InVals.push_back(Arg); 3812 3813 if (GPR_idx != Num_GPR_Regs) { 3814 unsigned VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass); 3815 FuncInfo->addLiveInAttr(VReg, Flags); 3816 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 3817 SDValue Store; 3818 3819 if (ObjSize==1 || ObjSize==2 || ObjSize==4) { 3820 EVT ObjType = (ObjSize == 1 ? MVT::i8 : 3821 (ObjSize == 2 ? MVT::i16 : MVT::i32)); 3822 Store = DAG.getTruncStore(Val.getValue(1), dl, Val, Arg, 3823 MachinePointerInfo(&*FuncArg), ObjType); 3824 } else { 3825 // For sizes that don't fit a truncating store (3, 5, 6, 7), 3826 // store the whole register as-is to the parameter save area 3827 // slot. 3828 Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, 3829 MachinePointerInfo(&*FuncArg)); 3830 } 3831 3832 MemOps.push_back(Store); 3833 } 3834 // Whether we copied from a register or not, advance the offset 3835 // into the parameter save area by a full doubleword. 3836 ArgOffset += PtrByteSize; 3837 continue; 3838 } 3839 3840 // The value of the object is its address, which is the address of 3841 // its first stack doubleword. 3842 InVals.push_back(FIN); 3843 3844 // Store whatever pieces of the object are in registers to memory. 3845 for (unsigned j = 0; j < ArgSize; j += PtrByteSize) { 3846 if (GPR_idx == Num_GPR_Regs) 3847 break; 3848 3849 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 3850 FuncInfo->addLiveInAttr(VReg, Flags); 3851 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 3852 SDValue Addr = FIN; 3853 if (j) { 3854 SDValue Off = DAG.getConstant(j, dl, PtrVT); 3855 Addr = DAG.getNode(ISD::ADD, dl, Off.getValueType(), Addr, Off); 3856 } 3857 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, Addr, 3858 MachinePointerInfo(&*FuncArg, j)); 3859 MemOps.push_back(Store); 3860 ++GPR_idx; 3861 } 3862 ArgOffset += ArgSize; 3863 continue; 3864 } 3865 3866 switch (ObjectVT.getSimpleVT().SimpleTy) { 3867 default: llvm_unreachable("Unhandled argument type!"); 3868 case MVT::i1: 3869 case MVT::i32: 3870 case MVT::i64: 3871 if (Flags.isNest()) { 3872 // The 'nest' parameter, if any, is passed in R11. 3873 unsigned VReg = MF.addLiveIn(PPC::X11, &PPC::G8RCRegClass); 3874 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); 3875 3876 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1) 3877 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl); 3878 3879 break; 3880 } 3881 3882 // These can be scalar arguments or elements of an integer array type 3883 // passed directly. Clang may use those instead of "byval" aggregate 3884 // types to avoid forcing arguments to memory unnecessarily. 3885 if (GPR_idx != Num_GPR_Regs) { 3886 unsigned VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass); 3887 FuncInfo->addLiveInAttr(VReg, Flags); 3888 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); 3889 3890 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1) 3891 // PPC64 passes i8, i16, and i32 values in i64 registers. Promote 3892 // value to MVT::i64 and then truncate to the correct register size. 3893 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl); 3894 } else { 3895 if (CallConv == CallingConv::Fast) 3896 ComputeArgOffset(); 3897 3898 needsLoad = true; 3899 ArgSize = PtrByteSize; 3900 } 3901 if (CallConv != CallingConv::Fast || needsLoad) 3902 ArgOffset += 8; 3903 break; 3904 3905 case MVT::f32: 3906 case MVT::f64: 3907 // These can be scalar arguments or elements of a float array type 3908 // passed directly. The latter are used to implement ELFv2 homogenous 3909 // float aggregates. 3910 if (FPR_idx != Num_FPR_Regs) { 3911 unsigned VReg; 3912 3913 if (ObjectVT == MVT::f32) 3914 VReg = MF.addLiveIn(FPR[FPR_idx], 3915 Subtarget.hasP8Vector() 3916 ? &PPC::VSSRCRegClass 3917 : &PPC::F4RCRegClass); 3918 else 3919 VReg = MF.addLiveIn(FPR[FPR_idx], Subtarget.hasVSX() 3920 ? &PPC::VSFRCRegClass 3921 : &PPC::F8RCRegClass); 3922 3923 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); 3924 ++FPR_idx; 3925 } else if (GPR_idx != Num_GPR_Regs && CallConv != CallingConv::Fast) { 3926 // FIXME: We may want to re-enable this for CallingConv::Fast on the P8 3927 // once we support fp <-> gpr moves. 3928 3929 // This can only ever happen in the presence of f32 array types, 3930 // since otherwise we never run out of FPRs before running out 3931 // of GPRs. 3932 unsigned VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass); 3933 FuncInfo->addLiveInAttr(VReg, Flags); 3934 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); 3935 3936 if (ObjectVT == MVT::f32) { 3937 if ((ArgOffset % PtrByteSize) == (isLittleEndian ? 4 : 0)) 3938 ArgVal = DAG.getNode(ISD::SRL, dl, MVT::i64, ArgVal, 3939 DAG.getConstant(32, dl, MVT::i32)); 3940 ArgVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, ArgVal); 3941 } 3942 3943 ArgVal = DAG.getNode(ISD::BITCAST, dl, ObjectVT, ArgVal); 3944 } else { 3945 if (CallConv == CallingConv::Fast) 3946 ComputeArgOffset(); 3947 3948 needsLoad = true; 3949 } 3950 3951 // When passing an array of floats, the array occupies consecutive 3952 // space in the argument area; only round up to the next doubleword 3953 // at the end of the array. Otherwise, each float takes 8 bytes. 3954 if (CallConv != CallingConv::Fast || needsLoad) { 3955 ArgSize = Flags.isInConsecutiveRegs() ? ObjSize : PtrByteSize; 3956 ArgOffset += ArgSize; 3957 if (Flags.isInConsecutiveRegsLast()) 3958 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 3959 } 3960 break; 3961 case MVT::v4f32: 3962 case MVT::v4i32: 3963 case MVT::v8i16: 3964 case MVT::v16i8: 3965 case MVT::v2f64: 3966 case MVT::v2i64: 3967 case MVT::v1i128: 3968 case MVT::f128: 3969 if (!Subtarget.hasQPX()) { 3970 // These can be scalar arguments or elements of a vector array type 3971 // passed directly. The latter are used to implement ELFv2 homogenous 3972 // vector aggregates. 3973 if (VR_idx != Num_VR_Regs) { 3974 unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass); 3975 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); 3976 ++VR_idx; 3977 } else { 3978 if (CallConv == CallingConv::Fast) 3979 ComputeArgOffset(); 3980 needsLoad = true; 3981 } 3982 if (CallConv != CallingConv::Fast || needsLoad) 3983 ArgOffset += 16; 3984 break; 3985 } // not QPX 3986 3987 assert(ObjectVT.getSimpleVT().SimpleTy == MVT::v4f32 && 3988 "Invalid QPX parameter type"); 3989 LLVM_FALLTHROUGH; 3990 3991 case MVT::v4f64: 3992 case MVT::v4i1: 3993 // QPX vectors are treated like their scalar floating-point subregisters 3994 // (except that they're larger). 3995 unsigned Sz = ObjectVT.getSimpleVT().SimpleTy == MVT::v4f32 ? 16 : 32; 3996 if (QFPR_idx != Num_QFPR_Regs) { 3997 const TargetRegisterClass *RC; 3998 switch (ObjectVT.getSimpleVT().SimpleTy) { 3999 case MVT::v4f64: RC = &PPC::QFRCRegClass; break; 4000 case MVT::v4f32: RC = &PPC::QSRCRegClass; break; 4001 default: RC = &PPC::QBRCRegClass; break; 4002 } 4003 4004 unsigned VReg = MF.addLiveIn(QFPR[QFPR_idx], RC); 4005 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); 4006 ++QFPR_idx; 4007 } else { 4008 if (CallConv == CallingConv::Fast) 4009 ComputeArgOffset(); 4010 needsLoad = true; 4011 } 4012 if (CallConv != CallingConv::Fast || needsLoad) 4013 ArgOffset += Sz; 4014 break; 4015 } 4016 4017 // We need to load the argument to a virtual register if we determined 4018 // above that we ran out of physical registers of the appropriate type. 4019 if (needsLoad) { 4020 if (ObjSize < ArgSize && !isLittleEndian) 4021 CurArgOffset += ArgSize - ObjSize; 4022 int FI = MFI.CreateFixedObject(ObjSize, CurArgOffset, isImmutable); 4023 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 4024 ArgVal = DAG.getLoad(ObjectVT, dl, Chain, FIN, MachinePointerInfo()); 4025 } 4026 4027 InVals.push_back(ArgVal); 4028 } 4029 4030 // Area that is at least reserved in the caller of this function. 4031 unsigned MinReservedArea; 4032 if (HasParameterArea) 4033 MinReservedArea = std::max(ArgOffset, LinkageSize + 8 * PtrByteSize); 4034 else 4035 MinReservedArea = LinkageSize; 4036 4037 // Set the size that is at least reserved in caller of this function. Tail 4038 // call optimized functions' reserved stack space needs to be aligned so that 4039 // taking the difference between two stack areas will result in an aligned 4040 // stack. 4041 MinReservedArea = 4042 EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea); 4043 FuncInfo->setMinReservedArea(MinReservedArea); 4044 4045 // If the function takes variable number of arguments, make a frame index for 4046 // the start of the first vararg value... for expansion of llvm.va_start. 4047 if (isVarArg) { 4048 int Depth = ArgOffset; 4049 4050 FuncInfo->setVarArgsFrameIndex( 4051 MFI.CreateFixedObject(PtrByteSize, Depth, true)); 4052 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 4053 4054 // If this function is vararg, store any remaining integer argument regs 4055 // to their spots on the stack so that they may be loaded by dereferencing 4056 // the result of va_next. 4057 for (GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize; 4058 GPR_idx < Num_GPR_Regs; ++GPR_idx) { 4059 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 4060 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 4061 SDValue Store = 4062 DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo()); 4063 MemOps.push_back(Store); 4064 // Increment the address by four for the next argument to store 4065 SDValue PtrOff = DAG.getConstant(PtrByteSize, dl, PtrVT); 4066 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); 4067 } 4068 } 4069 4070 if (!MemOps.empty()) 4071 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); 4072 4073 return Chain; 4074 } 4075 4076 SDValue PPCTargetLowering::LowerFormalArguments_Darwin( 4077 SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 4078 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 4079 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 4080 // TODO: add description of PPC stack frame format, or at least some docs. 4081 // 4082 MachineFunction &MF = DAG.getMachineFunction(); 4083 MachineFrameInfo &MFI = MF.getFrameInfo(); 4084 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 4085 4086 EVT PtrVT = getPointerTy(MF.getDataLayout()); 4087 bool isPPC64 = PtrVT == MVT::i64; 4088 // Potential tail calls could cause overwriting of argument stack slots. 4089 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && 4090 (CallConv == CallingConv::Fast)); 4091 unsigned PtrByteSize = isPPC64 ? 8 : 4; 4092 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 4093 unsigned ArgOffset = LinkageSize; 4094 // Area that is at least reserved in caller of this function. 4095 unsigned MinReservedArea = ArgOffset; 4096 4097 static const MCPhysReg GPR_32[] = { // 32-bit registers. 4098 PPC::R3, PPC::R4, PPC::R5, PPC::R6, 4099 PPC::R7, PPC::R8, PPC::R9, PPC::R10, 4100 }; 4101 static const MCPhysReg GPR_64[] = { // 64-bit registers. 4102 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 4103 PPC::X7, PPC::X8, PPC::X9, PPC::X10, 4104 }; 4105 static const MCPhysReg VR[] = { 4106 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, 4107 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 4108 }; 4109 4110 const unsigned Num_GPR_Regs = array_lengthof(GPR_32); 4111 const unsigned Num_FPR_Regs = useSoftFloat() ? 0 : 13; 4112 const unsigned Num_VR_Regs = array_lengthof( VR); 4113 4114 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; 4115 4116 const MCPhysReg *GPR = isPPC64 ? GPR_64 : GPR_32; 4117 4118 // In 32-bit non-varargs functions, the stack space for vectors is after the 4119 // stack space for non-vectors. We do not use this space unless we have 4120 // too many vectors to fit in registers, something that only occurs in 4121 // constructed examples:), but we have to walk the arglist to figure 4122 // that out...for the pathological case, compute VecArgOffset as the 4123 // start of the vector parameter area. Computing VecArgOffset is the 4124 // entire point of the following loop. 4125 unsigned VecArgOffset = ArgOffset; 4126 if (!isVarArg && !isPPC64) { 4127 for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; 4128 ++ArgNo) { 4129 EVT ObjectVT = Ins[ArgNo].VT; 4130 ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags; 4131 4132 if (Flags.isByVal()) { 4133 // ObjSize is the true size, ArgSize rounded up to multiple of regs. 4134 unsigned ObjSize = Flags.getByValSize(); 4135 unsigned ArgSize = 4136 ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 4137 VecArgOffset += ArgSize; 4138 continue; 4139 } 4140 4141 switch(ObjectVT.getSimpleVT().SimpleTy) { 4142 default: llvm_unreachable("Unhandled argument type!"); 4143 case MVT::i1: 4144 case MVT::i32: 4145 case MVT::f32: 4146 VecArgOffset += 4; 4147 break; 4148 case MVT::i64: // PPC64 4149 case MVT::f64: 4150 // FIXME: We are guaranteed to be !isPPC64 at this point. 4151 // Does MVT::i64 apply? 4152 VecArgOffset += 8; 4153 break; 4154 case MVT::v4f32: 4155 case MVT::v4i32: 4156 case MVT::v8i16: 4157 case MVT::v16i8: 4158 // Nothing to do, we're only looking at Nonvector args here. 4159 break; 4160 } 4161 } 4162 } 4163 // We've found where the vector parameter area in memory is. Skip the 4164 // first 12 parameters; these don't use that memory. 4165 VecArgOffset = ((VecArgOffset+15)/16)*16; 4166 VecArgOffset += 12*16; 4167 4168 // Add DAG nodes to load the arguments or copy them out of registers. On 4169 // entry to a function on PPC, the arguments start after the linkage area, 4170 // although the first ones are often in registers. 4171 4172 SmallVector<SDValue, 8> MemOps; 4173 unsigned nAltivecParamsAtEnd = 0; 4174 Function::const_arg_iterator FuncArg = MF.getFunction().arg_begin(); 4175 unsigned CurArgIdx = 0; 4176 for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) { 4177 SDValue ArgVal; 4178 bool needsLoad = false; 4179 EVT ObjectVT = Ins[ArgNo].VT; 4180 unsigned ObjSize = ObjectVT.getSizeInBits()/8; 4181 unsigned ArgSize = ObjSize; 4182 ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags; 4183 if (Ins[ArgNo].isOrigArg()) { 4184 std::advance(FuncArg, Ins[ArgNo].getOrigArgIndex() - CurArgIdx); 4185 CurArgIdx = Ins[ArgNo].getOrigArgIndex(); 4186 } 4187 unsigned CurArgOffset = ArgOffset; 4188 4189 // Varargs or 64 bit Altivec parameters are padded to a 16 byte boundary. 4190 if (ObjectVT==MVT::v4f32 || ObjectVT==MVT::v4i32 || 4191 ObjectVT==MVT::v8i16 || ObjectVT==MVT::v16i8) { 4192 if (isVarArg || isPPC64) { 4193 MinReservedArea = ((MinReservedArea+15)/16)*16; 4194 MinReservedArea += CalculateStackSlotSize(ObjectVT, 4195 Flags, 4196 PtrByteSize); 4197 } else nAltivecParamsAtEnd++; 4198 } else 4199 // Calculate min reserved area. 4200 MinReservedArea += CalculateStackSlotSize(Ins[ArgNo].VT, 4201 Flags, 4202 PtrByteSize); 4203 4204 // FIXME the codegen can be much improved in some cases. 4205 // We do not have to keep everything in memory. 4206 if (Flags.isByVal()) { 4207 assert(Ins[ArgNo].isOrigArg() && "Byval arguments cannot be implicit"); 4208 4209 // ObjSize is the true size, ArgSize rounded up to multiple of registers. 4210 ObjSize = Flags.getByValSize(); 4211 ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 4212 // Objects of size 1 and 2 are right justified, everything else is 4213 // left justified. This means the memory address is adjusted forwards. 4214 if (ObjSize==1 || ObjSize==2) { 4215 CurArgOffset = CurArgOffset + (4 - ObjSize); 4216 } 4217 // The value of the object is its address. 4218 int FI = MFI.CreateFixedObject(ObjSize, CurArgOffset, false, true); 4219 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 4220 InVals.push_back(FIN); 4221 if (ObjSize==1 || ObjSize==2) { 4222 if (GPR_idx != Num_GPR_Regs) { 4223 unsigned VReg; 4224 if (isPPC64) 4225 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 4226 else 4227 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); 4228 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 4229 EVT ObjType = ObjSize == 1 ? MVT::i8 : MVT::i16; 4230 SDValue Store = 4231 DAG.getTruncStore(Val.getValue(1), dl, Val, FIN, 4232 MachinePointerInfo(&*FuncArg), ObjType); 4233 MemOps.push_back(Store); 4234 ++GPR_idx; 4235 } 4236 4237 ArgOffset += PtrByteSize; 4238 4239 continue; 4240 } 4241 for (unsigned j = 0; j < ArgSize; j += PtrByteSize) { 4242 // Store whatever pieces of the object are in registers 4243 // to memory. ArgOffset will be the address of the beginning 4244 // of the object. 4245 if (GPR_idx != Num_GPR_Regs) { 4246 unsigned VReg; 4247 if (isPPC64) 4248 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 4249 else 4250 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); 4251 int FI = MFI.CreateFixedObject(PtrByteSize, ArgOffset, true); 4252 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 4253 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 4254 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, 4255 MachinePointerInfo(&*FuncArg, j)); 4256 MemOps.push_back(Store); 4257 ++GPR_idx; 4258 ArgOffset += PtrByteSize; 4259 } else { 4260 ArgOffset += ArgSize - (ArgOffset-CurArgOffset); 4261 break; 4262 } 4263 } 4264 continue; 4265 } 4266 4267 switch (ObjectVT.getSimpleVT().SimpleTy) { 4268 default: llvm_unreachable("Unhandled argument type!"); 4269 case MVT::i1: 4270 case MVT::i32: 4271 if (!isPPC64) { 4272 if (GPR_idx != Num_GPR_Regs) { 4273 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); 4274 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32); 4275 4276 if (ObjectVT == MVT::i1) 4277 ArgVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, ArgVal); 4278 4279 ++GPR_idx; 4280 } else { 4281 needsLoad = true; 4282 ArgSize = PtrByteSize; 4283 } 4284 // All int arguments reserve stack space in the Darwin ABI. 4285 ArgOffset += PtrByteSize; 4286 break; 4287 } 4288 LLVM_FALLTHROUGH; 4289 case MVT::i64: // PPC64 4290 if (GPR_idx != Num_GPR_Regs) { 4291 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 4292 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); 4293 4294 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1) 4295 // PPC64 passes i8, i16, and i32 values in i64 registers. Promote 4296 // value to MVT::i64 and then truncate to the correct register size. 4297 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl); 4298 4299 ++GPR_idx; 4300 } else { 4301 needsLoad = true; 4302 ArgSize = PtrByteSize; 4303 } 4304 // All int arguments reserve stack space in the Darwin ABI. 4305 ArgOffset += 8; 4306 break; 4307 4308 case MVT::f32: 4309 case MVT::f64: 4310 // Every 4 bytes of argument space consumes one of the GPRs available for 4311 // argument passing. 4312 if (GPR_idx != Num_GPR_Regs) { 4313 ++GPR_idx; 4314 if (ObjSize == 8 && GPR_idx != Num_GPR_Regs && !isPPC64) 4315 ++GPR_idx; 4316 } 4317 if (FPR_idx != Num_FPR_Regs) { 4318 unsigned VReg; 4319 4320 if (ObjectVT == MVT::f32) 4321 VReg = MF.addLiveIn(FPR[FPR_idx], &PPC::F4RCRegClass); 4322 else 4323 VReg = MF.addLiveIn(FPR[FPR_idx], &PPC::F8RCRegClass); 4324 4325 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); 4326 ++FPR_idx; 4327 } else { 4328 needsLoad = true; 4329 } 4330 4331 // All FP arguments reserve stack space in the Darwin ABI. 4332 ArgOffset += isPPC64 ? 8 : ObjSize; 4333 break; 4334 case MVT::v4f32: 4335 case MVT::v4i32: 4336 case MVT::v8i16: 4337 case MVT::v16i8: 4338 // Note that vector arguments in registers don't reserve stack space, 4339 // except in varargs functions. 4340 if (VR_idx != Num_VR_Regs) { 4341 unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass); 4342 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); 4343 if (isVarArg) { 4344 while ((ArgOffset % 16) != 0) { 4345 ArgOffset += PtrByteSize; 4346 if (GPR_idx != Num_GPR_Regs) 4347 GPR_idx++; 4348 } 4349 ArgOffset += 16; 4350 GPR_idx = std::min(GPR_idx+4, Num_GPR_Regs); // FIXME correct for ppc64? 4351 } 4352 ++VR_idx; 4353 } else { 4354 if (!isVarArg && !isPPC64) { 4355 // Vectors go after all the nonvectors. 4356 CurArgOffset = VecArgOffset; 4357 VecArgOffset += 16; 4358 } else { 4359 // Vectors are aligned. 4360 ArgOffset = ((ArgOffset+15)/16)*16; 4361 CurArgOffset = ArgOffset; 4362 ArgOffset += 16; 4363 } 4364 needsLoad = true; 4365 } 4366 break; 4367 } 4368 4369 // We need to load the argument to a virtual register if we determined above 4370 // that we ran out of physical registers of the appropriate type. 4371 if (needsLoad) { 4372 int FI = MFI.CreateFixedObject(ObjSize, 4373 CurArgOffset + (ArgSize - ObjSize), 4374 isImmutable); 4375 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 4376 ArgVal = DAG.getLoad(ObjectVT, dl, Chain, FIN, MachinePointerInfo()); 4377 } 4378 4379 InVals.push_back(ArgVal); 4380 } 4381 4382 // Allow for Altivec parameters at the end, if needed. 4383 if (nAltivecParamsAtEnd) { 4384 MinReservedArea = ((MinReservedArea+15)/16)*16; 4385 MinReservedArea += 16*nAltivecParamsAtEnd; 4386 } 4387 4388 // Area that is at least reserved in the caller of this function. 4389 MinReservedArea = std::max(MinReservedArea, LinkageSize + 8 * PtrByteSize); 4390 4391 // Set the size that is at least reserved in caller of this function. Tail 4392 // call optimized functions' reserved stack space needs to be aligned so that 4393 // taking the difference between two stack areas will result in an aligned 4394 // stack. 4395 MinReservedArea = 4396 EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea); 4397 FuncInfo->setMinReservedArea(MinReservedArea); 4398 4399 // If the function takes variable number of arguments, make a frame index for 4400 // the start of the first vararg value... for expansion of llvm.va_start. 4401 if (isVarArg) { 4402 int Depth = ArgOffset; 4403 4404 FuncInfo->setVarArgsFrameIndex( 4405 MFI.CreateFixedObject(PtrVT.getSizeInBits()/8, 4406 Depth, true)); 4407 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 4408 4409 // If this function is vararg, store any remaining integer argument regs 4410 // to their spots on the stack so that they may be loaded by dereferencing 4411 // the result of va_next. 4412 for (; GPR_idx != Num_GPR_Regs; ++GPR_idx) { 4413 unsigned VReg; 4414 4415 if (isPPC64) 4416 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 4417 else 4418 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); 4419 4420 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 4421 SDValue Store = 4422 DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo()); 4423 MemOps.push_back(Store); 4424 // Increment the address by four for the next argument to store 4425 SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, dl, PtrVT); 4426 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); 4427 } 4428 } 4429 4430 if (!MemOps.empty()) 4431 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); 4432 4433 return Chain; 4434 } 4435 4436 /// CalculateTailCallSPDiff - Get the amount the stack pointer has to be 4437 /// adjusted to accommodate the arguments for the tailcall. 4438 static int CalculateTailCallSPDiff(SelectionDAG& DAG, bool isTailCall, 4439 unsigned ParamSize) { 4440 4441 if (!isTailCall) return 0; 4442 4443 PPCFunctionInfo *FI = DAG.getMachineFunction().getInfo<PPCFunctionInfo>(); 4444 unsigned CallerMinReservedArea = FI->getMinReservedArea(); 4445 int SPDiff = (int)CallerMinReservedArea - (int)ParamSize; 4446 // Remember only if the new adjustment is bigger. 4447 if (SPDiff < FI->getTailCallSPDelta()) 4448 FI->setTailCallSPDelta(SPDiff); 4449 4450 return SPDiff; 4451 } 4452 4453 static bool isFunctionGlobalAddress(SDValue Callee); 4454 4455 static bool 4456 callsShareTOCBase(const Function *Caller, SDValue Callee, 4457 const TargetMachine &TM) { 4458 // Callee is either a GlobalAddress or an ExternalSymbol. ExternalSymbols 4459 // don't have enough information to determine if the caller and calle share 4460 // the same TOC base, so we have to pessimistically assume they don't for 4461 // correctness. 4462 GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee); 4463 if (!G) 4464 return false; 4465 4466 const GlobalValue *GV = G->getGlobal(); 4467 // The medium and large code models are expected to provide a sufficiently 4468 // large TOC to provide all data addressing needs of a module with a 4469 // single TOC. Since each module will be addressed with a single TOC then we 4470 // only need to check that caller and callee don't cross dso boundaries. 4471 if (CodeModel::Medium == TM.getCodeModel() || 4472 CodeModel::Large == TM.getCodeModel()) 4473 return TM.shouldAssumeDSOLocal(*Caller->getParent(), GV); 4474 4475 // Otherwise we need to ensure callee and caller are in the same section, 4476 // since the linker may allocate multiple TOCs, and we don't know which 4477 // sections will belong to the same TOC base. 4478 4479 if (!GV->isStrongDefinitionForLinker()) 4480 return false; 4481 4482 // Any explicitly-specified sections and section prefixes must also match. 4483 // Also, if we're using -ffunction-sections, then each function is always in 4484 // a different section (the same is true for COMDAT functions). 4485 if (TM.getFunctionSections() || GV->hasComdat() || Caller->hasComdat() || 4486 GV->getSection() != Caller->getSection()) 4487 return false; 4488 if (const auto *F = dyn_cast<Function>(GV)) { 4489 if (F->getSectionPrefix() != Caller->getSectionPrefix()) 4490 return false; 4491 } 4492 4493 // If the callee might be interposed, then we can't assume the ultimate call 4494 // target will be in the same section. Even in cases where we can assume that 4495 // interposition won't happen, in any case where the linker might insert a 4496 // stub to allow for interposition, we must generate code as though 4497 // interposition might occur. To understand why this matters, consider a 4498 // situation where: a -> b -> c where the arrows indicate calls. b and c are 4499 // in the same section, but a is in a different module (i.e. has a different 4500 // TOC base pointer). If the linker allows for interposition between b and c, 4501 // then it will generate a stub for the call edge between b and c which will 4502 // save the TOC pointer into the designated stack slot allocated by b. If we 4503 // return true here, and therefore allow a tail call between b and c, that 4504 // stack slot won't exist and the b -> c stub will end up saving b'c TOC base 4505 // pointer into the stack slot allocated by a (where the a -> b stub saved 4506 // a's TOC base pointer). If we're not considering a tail call, but rather, 4507 // whether a nop is needed after the call instruction in b, because the linker 4508 // will insert a stub, it might complain about a missing nop if we omit it 4509 // (although many don't complain in this case). 4510 if (!TM.shouldAssumeDSOLocal(*Caller->getParent(), GV)) 4511 return false; 4512 4513 return true; 4514 } 4515 4516 static bool 4517 needStackSlotPassParameters(const PPCSubtarget &Subtarget, 4518 const SmallVectorImpl<ISD::OutputArg> &Outs) { 4519 assert(Subtarget.isSVR4ABI() && Subtarget.isPPC64()); 4520 4521 const unsigned PtrByteSize = 8; 4522 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 4523 4524 static const MCPhysReg GPR[] = { 4525 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 4526 PPC::X7, PPC::X8, PPC::X9, PPC::X10, 4527 }; 4528 static const MCPhysReg VR[] = { 4529 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, 4530 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 4531 }; 4532 4533 const unsigned NumGPRs = array_lengthof(GPR); 4534 const unsigned NumFPRs = 13; 4535 const unsigned NumVRs = array_lengthof(VR); 4536 const unsigned ParamAreaSize = NumGPRs * PtrByteSize; 4537 4538 unsigned NumBytes = LinkageSize; 4539 unsigned AvailableFPRs = NumFPRs; 4540 unsigned AvailableVRs = NumVRs; 4541 4542 for (const ISD::OutputArg& Param : Outs) { 4543 if (Param.Flags.isNest()) continue; 4544 4545 if (CalculateStackSlotUsed(Param.VT, Param.ArgVT, Param.Flags, 4546 PtrByteSize, LinkageSize, ParamAreaSize, 4547 NumBytes, AvailableFPRs, AvailableVRs, 4548 Subtarget.hasQPX())) 4549 return true; 4550 } 4551 return false; 4552 } 4553 4554 static bool 4555 hasSameArgumentList(const Function *CallerFn, ImmutableCallSite CS) { 4556 if (CS.arg_size() != CallerFn->arg_size()) 4557 return false; 4558 4559 ImmutableCallSite::arg_iterator CalleeArgIter = CS.arg_begin(); 4560 ImmutableCallSite::arg_iterator CalleeArgEnd = CS.arg_end(); 4561 Function::const_arg_iterator CallerArgIter = CallerFn->arg_begin(); 4562 4563 for (; CalleeArgIter != CalleeArgEnd; ++CalleeArgIter, ++CallerArgIter) { 4564 const Value* CalleeArg = *CalleeArgIter; 4565 const Value* CallerArg = &(*CallerArgIter); 4566 if (CalleeArg == CallerArg) 4567 continue; 4568 4569 // e.g. @caller([4 x i64] %a, [4 x i64] %b) { 4570 // tail call @callee([4 x i64] undef, [4 x i64] %b) 4571 // } 4572 // 1st argument of callee is undef and has the same type as caller. 4573 if (CalleeArg->getType() == CallerArg->getType() && 4574 isa<UndefValue>(CalleeArg)) 4575 continue; 4576 4577 return false; 4578 } 4579 4580 return true; 4581 } 4582 4583 // Returns true if TCO is possible between the callers and callees 4584 // calling conventions. 4585 static bool 4586 areCallingConvEligibleForTCO_64SVR4(CallingConv::ID CallerCC, 4587 CallingConv::ID CalleeCC) { 4588 // Tail calls are possible with fastcc and ccc. 4589 auto isTailCallableCC = [] (CallingConv::ID CC){ 4590 return CC == CallingConv::C || CC == CallingConv::Fast; 4591 }; 4592 if (!isTailCallableCC(CallerCC) || !isTailCallableCC(CalleeCC)) 4593 return false; 4594 4595 // We can safely tail call both fastcc and ccc callees from a c calling 4596 // convention caller. If the caller is fastcc, we may have less stack space 4597 // than a non-fastcc caller with the same signature so disable tail-calls in 4598 // that case. 4599 return CallerCC == CallingConv::C || CallerCC == CalleeCC; 4600 } 4601 4602 bool 4603 PPCTargetLowering::IsEligibleForTailCallOptimization_64SVR4( 4604 SDValue Callee, 4605 CallingConv::ID CalleeCC, 4606 ImmutableCallSite CS, 4607 bool isVarArg, 4608 const SmallVectorImpl<ISD::OutputArg> &Outs, 4609 const SmallVectorImpl<ISD::InputArg> &Ins, 4610 SelectionDAG& DAG) const { 4611 bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt; 4612 4613 if (DisableSCO && !TailCallOpt) return false; 4614 4615 // Variadic argument functions are not supported. 4616 if (isVarArg) return false; 4617 4618 auto &Caller = DAG.getMachineFunction().getFunction(); 4619 // Check that the calling conventions are compatible for tco. 4620 if (!areCallingConvEligibleForTCO_64SVR4(Caller.getCallingConv(), CalleeCC)) 4621 return false; 4622 4623 // Caller contains any byval parameter is not supported. 4624 if (any_of(Ins, [](const ISD::InputArg &IA) { return IA.Flags.isByVal(); })) 4625 return false; 4626 4627 // Callee contains any byval parameter is not supported, too. 4628 // Note: This is a quick work around, because in some cases, e.g. 4629 // caller's stack size > callee's stack size, we are still able to apply 4630 // sibling call optimization. For example, gcc is able to do SCO for caller1 4631 // in the following example, but not for caller2. 4632 // struct test { 4633 // long int a; 4634 // char ary[56]; 4635 // } gTest; 4636 // __attribute__((noinline)) int callee(struct test v, struct test *b) { 4637 // b->a = v.a; 4638 // return 0; 4639 // } 4640 // void caller1(struct test a, struct test c, struct test *b) { 4641 // callee(gTest, b); } 4642 // void caller2(struct test *b) { callee(gTest, b); } 4643 if (any_of(Outs, [](const ISD::OutputArg& OA) { return OA.Flags.isByVal(); })) 4644 return false; 4645 4646 // If callee and caller use different calling conventions, we cannot pass 4647 // parameters on stack since offsets for the parameter area may be different. 4648 if (Caller.getCallingConv() != CalleeCC && 4649 needStackSlotPassParameters(Subtarget, Outs)) 4650 return false; 4651 4652 // No TCO/SCO on indirect call because Caller have to restore its TOC 4653 if (!isFunctionGlobalAddress(Callee) && 4654 !isa<ExternalSymbolSDNode>(Callee)) 4655 return false; 4656 4657 // If the caller and callee potentially have different TOC bases then we 4658 // cannot tail call since we need to restore the TOC pointer after the call. 4659 // ref: https://bugzilla.mozilla.org/show_bug.cgi?id=973977 4660 if (!callsShareTOCBase(&Caller, Callee, getTargetMachine())) 4661 return false; 4662 4663 // TCO allows altering callee ABI, so we don't have to check further. 4664 if (CalleeCC == CallingConv::Fast && TailCallOpt) 4665 return true; 4666 4667 if (DisableSCO) return false; 4668 4669 // If callee use the same argument list that caller is using, then we can 4670 // apply SCO on this case. If it is not, then we need to check if callee needs 4671 // stack for passing arguments. 4672 if (!hasSameArgumentList(&Caller, CS) && 4673 needStackSlotPassParameters(Subtarget, Outs)) { 4674 return false; 4675 } 4676 4677 return true; 4678 } 4679 4680 /// IsEligibleForTailCallOptimization - Check whether the call is eligible 4681 /// for tail call optimization. Targets which want to do tail call 4682 /// optimization should implement this function. 4683 bool 4684 PPCTargetLowering::IsEligibleForTailCallOptimization(SDValue Callee, 4685 CallingConv::ID CalleeCC, 4686 bool isVarArg, 4687 const SmallVectorImpl<ISD::InputArg> &Ins, 4688 SelectionDAG& DAG) const { 4689 if (!getTargetMachine().Options.GuaranteedTailCallOpt) 4690 return false; 4691 4692 // Variable argument functions are not supported. 4693 if (isVarArg) 4694 return false; 4695 4696 MachineFunction &MF = DAG.getMachineFunction(); 4697 CallingConv::ID CallerCC = MF.getFunction().getCallingConv(); 4698 if (CalleeCC == CallingConv::Fast && CallerCC == CalleeCC) { 4699 // Functions containing by val parameters are not supported. 4700 for (unsigned i = 0; i != Ins.size(); i++) { 4701 ISD::ArgFlagsTy Flags = Ins[i].Flags; 4702 if (Flags.isByVal()) return false; 4703 } 4704 4705 // Non-PIC/GOT tail calls are supported. 4706 if (getTargetMachine().getRelocationModel() != Reloc::PIC_) 4707 return true; 4708 4709 // At the moment we can only do local tail calls (in same module, hidden 4710 // or protected) if we are generating PIC. 4711 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) 4712 return G->getGlobal()->hasHiddenVisibility() 4713 || G->getGlobal()->hasProtectedVisibility(); 4714 } 4715 4716 return false; 4717 } 4718 4719 /// isCallCompatibleAddress - Return the immediate to use if the specified 4720 /// 32-bit value is representable in the immediate field of a BxA instruction. 4721 static SDNode *isBLACompatibleAddress(SDValue Op, SelectionDAG &DAG) { 4722 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op); 4723 if (!C) return nullptr; 4724 4725 int Addr = C->getZExtValue(); 4726 if ((Addr & 3) != 0 || // Low 2 bits are implicitly zero. 4727 SignExtend32<26>(Addr) != Addr) 4728 return nullptr; // Top 6 bits have to be sext of immediate. 4729 4730 return DAG 4731 .getConstant( 4732 (int)C->getZExtValue() >> 2, SDLoc(Op), 4733 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout())) 4734 .getNode(); 4735 } 4736 4737 namespace { 4738 4739 struct TailCallArgumentInfo { 4740 SDValue Arg; 4741 SDValue FrameIdxOp; 4742 int FrameIdx = 0; 4743 4744 TailCallArgumentInfo() = default; 4745 }; 4746 4747 } // end anonymous namespace 4748 4749 /// StoreTailCallArgumentsToStackSlot - Stores arguments to their stack slot. 4750 static void StoreTailCallArgumentsToStackSlot( 4751 SelectionDAG &DAG, SDValue Chain, 4752 const SmallVectorImpl<TailCallArgumentInfo> &TailCallArgs, 4753 SmallVectorImpl<SDValue> &MemOpChains, const SDLoc &dl) { 4754 for (unsigned i = 0, e = TailCallArgs.size(); i != e; ++i) { 4755 SDValue Arg = TailCallArgs[i].Arg; 4756 SDValue FIN = TailCallArgs[i].FrameIdxOp; 4757 int FI = TailCallArgs[i].FrameIdx; 4758 // Store relative to framepointer. 4759 MemOpChains.push_back(DAG.getStore( 4760 Chain, dl, Arg, FIN, 4761 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI))); 4762 } 4763 } 4764 4765 /// EmitTailCallStoreFPAndRetAddr - Move the frame pointer and return address to 4766 /// the appropriate stack slot for the tail call optimized function call. 4767 static SDValue EmitTailCallStoreFPAndRetAddr(SelectionDAG &DAG, SDValue Chain, 4768 SDValue OldRetAddr, SDValue OldFP, 4769 int SPDiff, const SDLoc &dl) { 4770 if (SPDiff) { 4771 // Calculate the new stack slot for the return address. 4772 MachineFunction &MF = DAG.getMachineFunction(); 4773 const PPCSubtarget &Subtarget = MF.getSubtarget<PPCSubtarget>(); 4774 const PPCFrameLowering *FL = Subtarget.getFrameLowering(); 4775 bool isPPC64 = Subtarget.isPPC64(); 4776 int SlotSize = isPPC64 ? 8 : 4; 4777 int NewRetAddrLoc = SPDiff + FL->getReturnSaveOffset(); 4778 int NewRetAddr = MF.getFrameInfo().CreateFixedObject(SlotSize, 4779 NewRetAddrLoc, true); 4780 EVT VT = isPPC64 ? MVT::i64 : MVT::i32; 4781 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewRetAddr, VT); 4782 Chain = DAG.getStore(Chain, dl, OldRetAddr, NewRetAddrFrIdx, 4783 MachinePointerInfo::getFixedStack(MF, NewRetAddr)); 4784 4785 // When using the 32/64-bit SVR4 ABI there is no need to move the FP stack 4786 // slot as the FP is never overwritten. 4787 if (Subtarget.isDarwinABI()) { 4788 int NewFPLoc = SPDiff + FL->getFramePointerSaveOffset(); 4789 int NewFPIdx = MF.getFrameInfo().CreateFixedObject(SlotSize, NewFPLoc, 4790 true); 4791 SDValue NewFramePtrIdx = DAG.getFrameIndex(NewFPIdx, VT); 4792 Chain = DAG.getStore(Chain, dl, OldFP, NewFramePtrIdx, 4793 MachinePointerInfo::getFixedStack( 4794 DAG.getMachineFunction(), NewFPIdx)); 4795 } 4796 } 4797 return Chain; 4798 } 4799 4800 /// CalculateTailCallArgDest - Remember Argument for later processing. Calculate 4801 /// the position of the argument. 4802 static void 4803 CalculateTailCallArgDest(SelectionDAG &DAG, MachineFunction &MF, bool isPPC64, 4804 SDValue Arg, int SPDiff, unsigned ArgOffset, 4805 SmallVectorImpl<TailCallArgumentInfo>& TailCallArguments) { 4806 int Offset = ArgOffset + SPDiff; 4807 uint32_t OpSize = (Arg.getValueSizeInBits() + 7) / 8; 4808 int FI = MF.getFrameInfo().CreateFixedObject(OpSize, Offset, true); 4809 EVT VT = isPPC64 ? MVT::i64 : MVT::i32; 4810 SDValue FIN = DAG.getFrameIndex(FI, VT); 4811 TailCallArgumentInfo Info; 4812 Info.Arg = Arg; 4813 Info.FrameIdxOp = FIN; 4814 Info.FrameIdx = FI; 4815 TailCallArguments.push_back(Info); 4816 } 4817 4818 /// EmitTCFPAndRetAddrLoad - Emit load from frame pointer and return address 4819 /// stack slot. Returns the chain as result and the loaded frame pointers in 4820 /// LROpOut/FPOpout. Used when tail calling. 4821 SDValue PPCTargetLowering::EmitTailCallLoadFPAndRetAddr( 4822 SelectionDAG &DAG, int SPDiff, SDValue Chain, SDValue &LROpOut, 4823 SDValue &FPOpOut, const SDLoc &dl) const { 4824 if (SPDiff) { 4825 // Load the LR and FP stack slot for later adjusting. 4826 EVT VT = Subtarget.isPPC64() ? MVT::i64 : MVT::i32; 4827 LROpOut = getReturnAddrFrameIndex(DAG); 4828 LROpOut = DAG.getLoad(VT, dl, Chain, LROpOut, MachinePointerInfo()); 4829 Chain = SDValue(LROpOut.getNode(), 1); 4830 4831 // When using the 32/64-bit SVR4 ABI there is no need to load the FP stack 4832 // slot as the FP is never overwritten. 4833 if (Subtarget.isDarwinABI()) { 4834 FPOpOut = getFramePointerFrameIndex(DAG); 4835 FPOpOut = DAG.getLoad(VT, dl, Chain, FPOpOut, MachinePointerInfo()); 4836 Chain = SDValue(FPOpOut.getNode(), 1); 4837 } 4838 } 4839 return Chain; 4840 } 4841 4842 /// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified 4843 /// by "Src" to address "Dst" of size "Size". Alignment information is 4844 /// specified by the specific parameter attribute. The copy will be passed as 4845 /// a byval function parameter. 4846 /// Sometimes what we are copying is the end of a larger object, the part that 4847 /// does not fit in registers. 4848 static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst, 4849 SDValue Chain, ISD::ArgFlagsTy Flags, 4850 SelectionDAG &DAG, const SDLoc &dl) { 4851 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32); 4852 return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(), 4853 false, false, false, MachinePointerInfo(), 4854 MachinePointerInfo()); 4855 } 4856 4857 /// LowerMemOpCallTo - Store the argument to the stack or remember it in case of 4858 /// tail calls. 4859 static void LowerMemOpCallTo( 4860 SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue Arg, 4861 SDValue PtrOff, int SPDiff, unsigned ArgOffset, bool isPPC64, 4862 bool isTailCall, bool isVector, SmallVectorImpl<SDValue> &MemOpChains, 4863 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments, const SDLoc &dl) { 4864 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); 4865 if (!isTailCall) { 4866 if (isVector) { 4867 SDValue StackPtr; 4868 if (isPPC64) 4869 StackPtr = DAG.getRegister(PPC::X1, MVT::i64); 4870 else 4871 StackPtr = DAG.getRegister(PPC::R1, MVT::i32); 4872 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, 4873 DAG.getConstant(ArgOffset, dl, PtrVT)); 4874 } 4875 MemOpChains.push_back( 4876 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo())); 4877 // Calculate and remember argument location. 4878 } else CalculateTailCallArgDest(DAG, MF, isPPC64, Arg, SPDiff, ArgOffset, 4879 TailCallArguments); 4880 } 4881 4882 static void 4883 PrepareTailCall(SelectionDAG &DAG, SDValue &InFlag, SDValue &Chain, 4884 const SDLoc &dl, int SPDiff, unsigned NumBytes, SDValue LROp, 4885 SDValue FPOp, 4886 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments) { 4887 // Emit a sequence of copyto/copyfrom virtual registers for arguments that 4888 // might overwrite each other in case of tail call optimization. 4889 SmallVector<SDValue, 8> MemOpChains2; 4890 // Do not flag preceding copytoreg stuff together with the following stuff. 4891 InFlag = SDValue(); 4892 StoreTailCallArgumentsToStackSlot(DAG, Chain, TailCallArguments, 4893 MemOpChains2, dl); 4894 if (!MemOpChains2.empty()) 4895 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2); 4896 4897 // Store the return address to the appropriate stack slot. 4898 Chain = EmitTailCallStoreFPAndRetAddr(DAG, Chain, LROp, FPOp, SPDiff, dl); 4899 4900 // Emit callseq_end just before tailcall node. 4901 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true), 4902 DAG.getIntPtrConstant(0, dl, true), InFlag, dl); 4903 InFlag = Chain.getValue(1); 4904 } 4905 4906 // Is this global address that of a function that can be called by name? (as 4907 // opposed to something that must hold a descriptor for an indirect call). 4908 static bool isFunctionGlobalAddress(SDValue Callee) { 4909 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 4910 if (Callee.getOpcode() == ISD::GlobalTLSAddress || 4911 Callee.getOpcode() == ISD::TargetGlobalTLSAddress) 4912 return false; 4913 4914 return G->getGlobal()->getValueType()->isFunctionTy(); 4915 } 4916 4917 return false; 4918 } 4919 4920 static unsigned 4921 PrepareCall(SelectionDAG &DAG, SDValue &Callee, SDValue &InFlag, SDValue &Chain, 4922 SDValue CallSeqStart, const SDLoc &dl, int SPDiff, bool isTailCall, 4923 bool isPatchPoint, bool hasNest, 4924 SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass, 4925 SmallVectorImpl<SDValue> &Ops, std::vector<EVT> &NodeTys, 4926 ImmutableCallSite CS, const PPCSubtarget &Subtarget) { 4927 bool isPPC64 = Subtarget.isPPC64(); 4928 bool isSVR4ABI = Subtarget.isSVR4ABI(); 4929 bool isELFv2ABI = Subtarget.isELFv2ABI(); 4930 bool isAIXABI = Subtarget.isAIXABI(); 4931 4932 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); 4933 NodeTys.push_back(MVT::Other); // Returns a chain 4934 NodeTys.push_back(MVT::Glue); // Returns a flag for retval copy to use. 4935 4936 unsigned CallOpc = PPCISD::CALL; 4937 4938 bool needIndirectCall = true; 4939 if (!isSVR4ABI || !isPPC64) 4940 if (SDNode *Dest = isBLACompatibleAddress(Callee, DAG)) { 4941 // If this is an absolute destination address, use the munged value. 4942 Callee = SDValue(Dest, 0); 4943 needIndirectCall = false; 4944 } 4945 4946 // PC-relative references to external symbols should go through $stub, unless 4947 // we're building with the leopard linker or later, which automatically 4948 // synthesizes these stubs. 4949 const TargetMachine &TM = DAG.getTarget(); 4950 const Module *Mod = DAG.getMachineFunction().getFunction().getParent(); 4951 const GlobalValue *GV = nullptr; 4952 if (auto *G = dyn_cast<GlobalAddressSDNode>(Callee)) 4953 GV = G->getGlobal(); 4954 bool Local = TM.shouldAssumeDSOLocal(*Mod, GV); 4955 // The PLT is only used in 32-bit ELF PIC mode. Attempting to use the PLT in 4956 // a static relocation model causes some versions of GNU LD (2.17.50, at 4957 // least) to force BSS-PLT, instead of secure-PLT, even if all objects are 4958 // built with secure-PLT. 4959 bool UsePlt = !Local && Subtarget.isTargetELF() && !isPPC64 && 4960 Subtarget.getTargetMachine().getRelocationModel() == Reloc::PIC_; 4961 4962 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, 4963 // every direct call is) turn it into a TargetGlobalAddress / 4964 // TargetExternalSymbol node so that legalize doesn't hack it. 4965 if (isFunctionGlobalAddress(Callee)) { 4966 GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Callee); 4967 4968 // A call to a TLS address is actually an indirect call to a 4969 // thread-specific pointer. 4970 unsigned OpFlags = 0; 4971 if (UsePlt) 4972 OpFlags = PPCII::MO_PLT; 4973 4974 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), dl, 4975 Callee.getValueType(), 0, OpFlags); 4976 needIndirectCall = false; 4977 } 4978 4979 if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) { 4980 unsigned char OpFlags = 0; 4981 4982 if (UsePlt) 4983 OpFlags = PPCII::MO_PLT; 4984 4985 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), Callee.getValueType(), 4986 OpFlags); 4987 needIndirectCall = false; 4988 } 4989 4990 if (isPatchPoint) { 4991 // We'll form an invalid direct call when lowering a patchpoint; the full 4992 // sequence for an indirect call is complicated, and many of the 4993 // instructions introduced might have side effects (and, thus, can't be 4994 // removed later). The call itself will be removed as soon as the 4995 // argument/return lowering is complete, so the fact that it has the wrong 4996 // kind of operands should not really matter. 4997 needIndirectCall = false; 4998 } 4999 5000 if (needIndirectCall) { 5001 // Otherwise, this is an indirect call. We have to use a MTCTR/BCTRL pair 5002 // to do the call, we can't use PPCISD::CALL. 5003 SDValue MTCTROps[] = {Chain, Callee, InFlag}; 5004 5005 if (isSVR4ABI && isPPC64 && !isELFv2ABI) { 5006 // Function pointers in the 64-bit SVR4 ABI do not point to the function 5007 // entry point, but to the function descriptor (the function entry point 5008 // address is part of the function descriptor though). 5009 // The function descriptor is a three doubleword structure with the 5010 // following fields: function entry point, TOC base address and 5011 // environment pointer. 5012 // Thus for a call through a function pointer, the following actions need 5013 // to be performed: 5014 // 1. Save the TOC of the caller in the TOC save area of its stack 5015 // frame (this is done in LowerCall_Darwin() or LowerCall_64SVR4()). 5016 // 2. Load the address of the function entry point from the function 5017 // descriptor. 5018 // 3. Load the TOC of the callee from the function descriptor into r2. 5019 // 4. Load the environment pointer from the function descriptor into 5020 // r11. 5021 // 5. Branch to the function entry point address. 5022 // 6. On return of the callee, the TOC of the caller needs to be 5023 // restored (this is done in FinishCall()). 5024 // 5025 // The loads are scheduled at the beginning of the call sequence, and the 5026 // register copies are flagged together to ensure that no other 5027 // operations can be scheduled in between. E.g. without flagging the 5028 // copies together, a TOC access in the caller could be scheduled between 5029 // the assignment of the callee TOC and the branch to the callee, which 5030 // results in the TOC access going through the TOC of the callee instead 5031 // of going through the TOC of the caller, which leads to incorrect code. 5032 5033 // Load the address of the function entry point from the function 5034 // descriptor. 5035 SDValue LDChain = CallSeqStart.getValue(CallSeqStart->getNumValues()-1); 5036 if (LDChain.getValueType() == MVT::Glue) 5037 LDChain = CallSeqStart.getValue(CallSeqStart->getNumValues()-2); 5038 5039 auto MMOFlags = Subtarget.hasInvariantFunctionDescriptors() 5040 ? (MachineMemOperand::MODereferenceable | 5041 MachineMemOperand::MOInvariant) 5042 : MachineMemOperand::MONone; 5043 5044 MachinePointerInfo MPI(CS ? CS.getCalledValue() : nullptr); 5045 SDValue LoadFuncPtr = DAG.getLoad(MVT::i64, dl, LDChain, Callee, MPI, 5046 /* Alignment = */ 8, MMOFlags); 5047 5048 // Load environment pointer into r11. 5049 SDValue PtrOff = DAG.getIntPtrConstant(16, dl); 5050 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, MVT::i64, Callee, PtrOff); 5051 SDValue LoadEnvPtr = 5052 DAG.getLoad(MVT::i64, dl, LDChain, AddPtr, MPI.getWithOffset(16), 5053 /* Alignment = */ 8, MMOFlags); 5054 5055 SDValue TOCOff = DAG.getIntPtrConstant(8, dl); 5056 SDValue AddTOC = DAG.getNode(ISD::ADD, dl, MVT::i64, Callee, TOCOff); 5057 SDValue TOCPtr = 5058 DAG.getLoad(MVT::i64, dl, LDChain, AddTOC, MPI.getWithOffset(8), 5059 /* Alignment = */ 8, MMOFlags); 5060 5061 setUsesTOCBasePtr(DAG); 5062 SDValue TOCVal = DAG.getCopyToReg(Chain, dl, PPC::X2, TOCPtr, 5063 InFlag); 5064 Chain = TOCVal.getValue(0); 5065 InFlag = TOCVal.getValue(1); 5066 5067 // If the function call has an explicit 'nest' parameter, it takes the 5068 // place of the environment pointer. 5069 if (!hasNest) { 5070 SDValue EnvVal = DAG.getCopyToReg(Chain, dl, PPC::X11, LoadEnvPtr, 5071 InFlag); 5072 5073 Chain = EnvVal.getValue(0); 5074 InFlag = EnvVal.getValue(1); 5075 } 5076 5077 MTCTROps[0] = Chain; 5078 MTCTROps[1] = LoadFuncPtr; 5079 MTCTROps[2] = InFlag; 5080 } 5081 5082 Chain = DAG.getNode(PPCISD::MTCTR, dl, NodeTys, 5083 makeArrayRef(MTCTROps, InFlag.getNode() ? 3 : 2)); 5084 InFlag = Chain.getValue(1); 5085 5086 NodeTys.clear(); 5087 NodeTys.push_back(MVT::Other); 5088 NodeTys.push_back(MVT::Glue); 5089 Ops.push_back(Chain); 5090 CallOpc = PPCISD::BCTRL; 5091 Callee.setNode(nullptr); 5092 // Add use of X11 (holding environment pointer) 5093 if (isSVR4ABI && isPPC64 && !isELFv2ABI && !hasNest) 5094 Ops.push_back(DAG.getRegister(PPC::X11, PtrVT)); 5095 // Add CTR register as callee so a bctr can be emitted later. 5096 if (isTailCall) 5097 Ops.push_back(DAG.getRegister(isPPC64 ? PPC::CTR8 : PPC::CTR, PtrVT)); 5098 } 5099 5100 // If this is a direct call, pass the chain and the callee. 5101 if (Callee.getNode()) { 5102 Ops.push_back(Chain); 5103 Ops.push_back(Callee); 5104 } 5105 // If this is a tail call add stack pointer delta. 5106 if (isTailCall) 5107 Ops.push_back(DAG.getConstant(SPDiff, dl, MVT::i32)); 5108 5109 // Add argument registers to the end of the list so that they are known live 5110 // into the call. 5111 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) 5112 Ops.push_back(DAG.getRegister(RegsToPass[i].first, 5113 RegsToPass[i].second.getValueType())); 5114 5115 // All calls, in the AIX ABI and 64-bit ELF ABIs, need the TOC register 5116 // live into the call. 5117 // We do need to reserve R2/X2 to appease the verifier for the PATCHPOINT. 5118 if ((isSVR4ABI && isPPC64) || isAIXABI) { 5119 setUsesTOCBasePtr(DAG); 5120 5121 // We cannot add R2/X2 as an operand here for PATCHPOINT, because there is 5122 // no way to mark dependencies as implicit here. 5123 // We will add the R2/X2 dependency in EmitInstrWithCustomInserter. 5124 if (!isPatchPoint) 5125 Ops.push_back(DAG.getRegister(isPPC64 ? PPC::X2 5126 : PPC::R2, PtrVT)); 5127 } 5128 5129 return CallOpc; 5130 } 5131 5132 SDValue PPCTargetLowering::LowerCallResult( 5133 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg, 5134 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 5135 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 5136 SmallVector<CCValAssign, 16> RVLocs; 5137 CCState CCRetInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, 5138 *DAG.getContext()); 5139 5140 CCRetInfo.AnalyzeCallResult( 5141 Ins, (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold) 5142 ? RetCC_PPC_Cold 5143 : RetCC_PPC); 5144 5145 // Copy all of the result registers out of their specified physreg. 5146 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) { 5147 CCValAssign &VA = RVLocs[i]; 5148 assert(VA.isRegLoc() && "Can only return in registers!"); 5149 5150 SDValue Val; 5151 5152 if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) { 5153 SDValue Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, 5154 InFlag); 5155 Chain = Lo.getValue(1); 5156 InFlag = Lo.getValue(2); 5157 VA = RVLocs[++i]; // skip ahead to next loc 5158 SDValue Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, 5159 InFlag); 5160 Chain = Hi.getValue(1); 5161 InFlag = Hi.getValue(2); 5162 if (!Subtarget.isLittleEndian()) 5163 std::swap (Lo, Hi); 5164 Val = DAG.getNode(PPCISD::BUILD_SPE64, dl, MVT::f64, Lo, Hi); 5165 } else { 5166 Val = DAG.getCopyFromReg(Chain, dl, 5167 VA.getLocReg(), VA.getLocVT(), InFlag); 5168 Chain = Val.getValue(1); 5169 InFlag = Val.getValue(2); 5170 } 5171 5172 switch (VA.getLocInfo()) { 5173 default: llvm_unreachable("Unknown loc info!"); 5174 case CCValAssign::Full: break; 5175 case CCValAssign::AExt: 5176 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); 5177 break; 5178 case CCValAssign::ZExt: 5179 Val = DAG.getNode(ISD::AssertZext, dl, VA.getLocVT(), Val, 5180 DAG.getValueType(VA.getValVT())); 5181 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); 5182 break; 5183 case CCValAssign::SExt: 5184 Val = DAG.getNode(ISD::AssertSext, dl, VA.getLocVT(), Val, 5185 DAG.getValueType(VA.getValVT())); 5186 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); 5187 break; 5188 } 5189 5190 InVals.push_back(Val); 5191 } 5192 5193 return Chain; 5194 } 5195 5196 SDValue PPCTargetLowering::FinishCall( 5197 CallingConv::ID CallConv, const SDLoc &dl, bool isTailCall, bool isVarArg, 5198 bool isPatchPoint, bool hasNest, SelectionDAG &DAG, 5199 SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass, SDValue InFlag, 5200 SDValue Chain, SDValue CallSeqStart, SDValue &Callee, int SPDiff, 5201 unsigned NumBytes, const SmallVectorImpl<ISD::InputArg> &Ins, 5202 SmallVectorImpl<SDValue> &InVals, ImmutableCallSite CS) const { 5203 std::vector<EVT> NodeTys; 5204 SmallVector<SDValue, 8> Ops; 5205 unsigned CallOpc = PrepareCall(DAG, Callee, InFlag, Chain, CallSeqStart, dl, 5206 SPDiff, isTailCall, isPatchPoint, hasNest, 5207 RegsToPass, Ops, NodeTys, CS, Subtarget); 5208 5209 // Add implicit use of CR bit 6 for 32-bit SVR4 vararg calls 5210 if (isVarArg && Subtarget.isSVR4ABI() && !Subtarget.isPPC64()) 5211 Ops.push_back(DAG.getRegister(PPC::CR1EQ, MVT::i32)); 5212 5213 // When performing tail call optimization the callee pops its arguments off 5214 // the stack. Account for this here so these bytes can be pushed back on in 5215 // PPCFrameLowering::eliminateCallFramePseudoInstr. 5216 int BytesCalleePops = 5217 (CallConv == CallingConv::Fast && 5218 getTargetMachine().Options.GuaranteedTailCallOpt) ? NumBytes : 0; 5219 5220 // Add a register mask operand representing the call-preserved registers. 5221 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); 5222 const uint32_t *Mask = 5223 TRI->getCallPreservedMask(DAG.getMachineFunction(), CallConv); 5224 assert(Mask && "Missing call preserved mask for calling convention"); 5225 Ops.push_back(DAG.getRegisterMask(Mask)); 5226 5227 if (InFlag.getNode()) 5228 Ops.push_back(InFlag); 5229 5230 // Emit tail call. 5231 if (isTailCall) { 5232 assert(((Callee.getOpcode() == ISD::Register && 5233 cast<RegisterSDNode>(Callee)->getReg() == PPC::CTR) || 5234 Callee.getOpcode() == ISD::TargetExternalSymbol || 5235 Callee.getOpcode() == ISD::TargetGlobalAddress || 5236 isa<ConstantSDNode>(Callee)) && 5237 "Expecting an global address, external symbol, absolute value or register"); 5238 5239 DAG.getMachineFunction().getFrameInfo().setHasTailCall(); 5240 return DAG.getNode(PPCISD::TC_RETURN, dl, MVT::Other, Ops); 5241 } 5242 5243 // Add a NOP immediately after the branch instruction when using the 64-bit 5244 // SVR4 or the AIX ABI. 5245 // At link time, if caller and callee are in a different module and 5246 // thus have a different TOC, the call will be replaced with a call to a stub 5247 // function which saves the current TOC, loads the TOC of the callee and 5248 // branches to the callee. The NOP will be replaced with a load instruction 5249 // which restores the TOC of the caller from the TOC save slot of the current 5250 // stack frame. If caller and callee belong to the same module (and have the 5251 // same TOC), the NOP will remain unchanged, or become some other NOP. 5252 5253 MachineFunction &MF = DAG.getMachineFunction(); 5254 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 5255 if (!isTailCall && !isPatchPoint && 5256 ((Subtarget.isSVR4ABI() && Subtarget.isPPC64()) || 5257 Subtarget.isAIXABI())) { 5258 if (CallOpc == PPCISD::BCTRL) { 5259 if (Subtarget.isAIXABI()) 5260 report_fatal_error("Indirect call on AIX is not implemented."); 5261 5262 // This is a call through a function pointer. 5263 // Restore the caller TOC from the save area into R2. 5264 // See PrepareCall() for more information about calls through function 5265 // pointers in the 64-bit SVR4 ABI. 5266 // We are using a target-specific load with r2 hard coded, because the 5267 // result of a target-independent load would never go directly into r2, 5268 // since r2 is a reserved register (which prevents the register allocator 5269 // from allocating it), resulting in an additional register being 5270 // allocated and an unnecessary move instruction being generated. 5271 CallOpc = PPCISD::BCTRL_LOAD_TOC; 5272 5273 SDValue StackPtr = DAG.getRegister(PPC::X1, PtrVT); 5274 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset(); 5275 SDValue TOCOff = DAG.getIntPtrConstant(TOCSaveOffset, dl); 5276 SDValue AddTOC = DAG.getNode(ISD::ADD, dl, MVT::i64, StackPtr, TOCOff); 5277 5278 // The address needs to go after the chain input but before the flag (or 5279 // any other variadic arguments). 5280 Ops.insert(std::next(Ops.begin()), AddTOC); 5281 } else if (CallOpc == PPCISD::CALL && 5282 !callsShareTOCBase(&MF.getFunction(), Callee, DAG.getTarget())) { 5283 // Otherwise insert NOP for non-local calls. 5284 CallOpc = PPCISD::CALL_NOP; 5285 } 5286 } 5287 5288 if (Subtarget.isAIXABI() && isFunctionGlobalAddress(Callee)) { 5289 // On AIX, direct function calls reference the symbol for the function's 5290 // entry point, which is named by inserting a "." before the function's 5291 // C-linkage name. 5292 GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Callee); 5293 auto &Context = DAG.getMachineFunction().getMMI().getContext(); 5294 MCSymbol *S = Context.getOrCreateSymbol(Twine(".") + 5295 Twine(G->getGlobal()->getName())); 5296 Callee = DAG.getMCSymbol(S, PtrVT); 5297 // Replace the GlobalAddressSDNode Callee with the MCSymbolSDNode. 5298 Ops[1] = Callee; 5299 } 5300 5301 Chain = DAG.getNode(CallOpc, dl, NodeTys, Ops); 5302 InFlag = Chain.getValue(1); 5303 5304 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true), 5305 DAG.getIntPtrConstant(BytesCalleePops, dl, true), 5306 InFlag, dl); 5307 if (!Ins.empty()) 5308 InFlag = Chain.getValue(1); 5309 5310 return LowerCallResult(Chain, InFlag, CallConv, isVarArg, 5311 Ins, dl, DAG, InVals); 5312 } 5313 5314 SDValue 5315 PPCTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, 5316 SmallVectorImpl<SDValue> &InVals) const { 5317 SelectionDAG &DAG = CLI.DAG; 5318 SDLoc &dl = CLI.DL; 5319 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; 5320 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; 5321 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; 5322 SDValue Chain = CLI.Chain; 5323 SDValue Callee = CLI.Callee; 5324 bool &isTailCall = CLI.IsTailCall; 5325 CallingConv::ID CallConv = CLI.CallConv; 5326 bool isVarArg = CLI.IsVarArg; 5327 bool isPatchPoint = CLI.IsPatchPoint; 5328 ImmutableCallSite CS = CLI.CS; 5329 5330 if (isTailCall) { 5331 if (Subtarget.useLongCalls() && !(CS && CS.isMustTailCall())) 5332 isTailCall = false; 5333 else if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) 5334 isTailCall = 5335 IsEligibleForTailCallOptimization_64SVR4(Callee, CallConv, CS, 5336 isVarArg, Outs, Ins, DAG); 5337 else 5338 isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv, isVarArg, 5339 Ins, DAG); 5340 if (isTailCall) { 5341 ++NumTailCalls; 5342 if (!getTargetMachine().Options.GuaranteedTailCallOpt) 5343 ++NumSiblingCalls; 5344 5345 assert(isa<GlobalAddressSDNode>(Callee) && 5346 "Callee should be an llvm::Function object."); 5347 LLVM_DEBUG( 5348 const GlobalValue *GV = 5349 cast<GlobalAddressSDNode>(Callee)->getGlobal(); 5350 const unsigned Width = 5351 80 - strlen("TCO caller: ") - strlen(", callee linkage: 0, 0"); 5352 dbgs() << "TCO caller: " 5353 << left_justify(DAG.getMachineFunction().getName(), Width) 5354 << ", callee linkage: " << GV->getVisibility() << ", " 5355 << GV->getLinkage() << "\n"); 5356 } 5357 } 5358 5359 if (!isTailCall && CS && CS.isMustTailCall()) 5360 report_fatal_error("failed to perform tail call elimination on a call " 5361 "site marked musttail"); 5362 5363 // When long calls (i.e. indirect calls) are always used, calls are always 5364 // made via function pointer. If we have a function name, first translate it 5365 // into a pointer. 5366 if (Subtarget.useLongCalls() && isa<GlobalAddressSDNode>(Callee) && 5367 !isTailCall) 5368 Callee = LowerGlobalAddress(Callee, DAG); 5369 5370 if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) 5371 return LowerCall_64SVR4(Chain, Callee, CallConv, isVarArg, 5372 isTailCall, isPatchPoint, Outs, OutVals, Ins, 5373 dl, DAG, InVals, CS); 5374 5375 if (Subtarget.isSVR4ABI()) 5376 return LowerCall_32SVR4(Chain, Callee, CallConv, isVarArg, 5377 isTailCall, isPatchPoint, Outs, OutVals, Ins, 5378 dl, DAG, InVals, CS); 5379 5380 if (Subtarget.isAIXABI()) 5381 return LowerCall_AIX(Chain, Callee, CallConv, isVarArg, 5382 isTailCall, isPatchPoint, Outs, OutVals, Ins, 5383 dl, DAG, InVals, CS); 5384 5385 return LowerCall_Darwin(Chain, Callee, CallConv, isVarArg, 5386 isTailCall, isPatchPoint, Outs, OutVals, Ins, 5387 dl, DAG, InVals, CS); 5388 } 5389 5390 SDValue PPCTargetLowering::LowerCall_32SVR4( 5391 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg, 5392 bool isTailCall, bool isPatchPoint, 5393 const SmallVectorImpl<ISD::OutputArg> &Outs, 5394 const SmallVectorImpl<SDValue> &OutVals, 5395 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 5396 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, 5397 ImmutableCallSite CS) const { 5398 // See PPCTargetLowering::LowerFormalArguments_32SVR4() for a description 5399 // of the 32-bit SVR4 ABI stack frame layout. 5400 5401 assert((CallConv == CallingConv::C || 5402 CallConv == CallingConv::Cold || 5403 CallConv == CallingConv::Fast) && "Unknown calling convention!"); 5404 5405 unsigned PtrByteSize = 4; 5406 5407 MachineFunction &MF = DAG.getMachineFunction(); 5408 5409 // Mark this function as potentially containing a function that contains a 5410 // tail call. As a consequence the frame pointer will be used for dynamicalloc 5411 // and restoring the callers stack pointer in this functions epilog. This is 5412 // done because by tail calling the called function might overwrite the value 5413 // in this function's (MF) stack pointer stack slot 0(SP). 5414 if (getTargetMachine().Options.GuaranteedTailCallOpt && 5415 CallConv == CallingConv::Fast) 5416 MF.getInfo<PPCFunctionInfo>()->setHasFastCall(); 5417 5418 // Count how many bytes are to be pushed on the stack, including the linkage 5419 // area, parameter list area and the part of the local variable space which 5420 // contains copies of aggregates which are passed by value. 5421 5422 // Assign locations to all of the outgoing arguments. 5423 SmallVector<CCValAssign, 16> ArgLocs; 5424 PPCCCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext()); 5425 5426 // Reserve space for the linkage area on the stack. 5427 CCInfo.AllocateStack(Subtarget.getFrameLowering()->getLinkageSize(), 5428 PtrByteSize); 5429 if (useSoftFloat()) 5430 CCInfo.PreAnalyzeCallOperands(Outs); 5431 5432 if (isVarArg) { 5433 // Handle fixed and variable vector arguments differently. 5434 // Fixed vector arguments go into registers as long as registers are 5435 // available. Variable vector arguments always go into memory. 5436 unsigned NumArgs = Outs.size(); 5437 5438 for (unsigned i = 0; i != NumArgs; ++i) { 5439 MVT ArgVT = Outs[i].VT; 5440 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; 5441 bool Result; 5442 5443 if (Outs[i].IsFixed) { 5444 Result = CC_PPC32_SVR4(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, 5445 CCInfo); 5446 } else { 5447 Result = CC_PPC32_SVR4_VarArg(i, ArgVT, ArgVT, CCValAssign::Full, 5448 ArgFlags, CCInfo); 5449 } 5450 5451 if (Result) { 5452 #ifndef NDEBUG 5453 errs() << "Call operand #" << i << " has unhandled type " 5454 << EVT(ArgVT).getEVTString() << "\n"; 5455 #endif 5456 llvm_unreachable(nullptr); 5457 } 5458 } 5459 } else { 5460 // All arguments are treated the same. 5461 CCInfo.AnalyzeCallOperands(Outs, CC_PPC32_SVR4); 5462 } 5463 CCInfo.clearWasPPCF128(); 5464 5465 // Assign locations to all of the outgoing aggregate by value arguments. 5466 SmallVector<CCValAssign, 16> ByValArgLocs; 5467 CCState CCByValInfo(CallConv, isVarArg, MF, ByValArgLocs, *DAG.getContext()); 5468 5469 // Reserve stack space for the allocations in CCInfo. 5470 CCByValInfo.AllocateStack(CCInfo.getNextStackOffset(), PtrByteSize); 5471 5472 CCByValInfo.AnalyzeCallOperands(Outs, CC_PPC32_SVR4_ByVal); 5473 5474 // Size of the linkage area, parameter list area and the part of the local 5475 // space variable where copies of aggregates which are passed by value are 5476 // stored. 5477 unsigned NumBytes = CCByValInfo.getNextStackOffset(); 5478 5479 // Calculate by how many bytes the stack has to be adjusted in case of tail 5480 // call optimization. 5481 int SPDiff = CalculateTailCallSPDiff(DAG, isTailCall, NumBytes); 5482 5483 // Adjust the stack pointer for the new arguments... 5484 // These operations are automatically eliminated by the prolog/epilog pass 5485 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); 5486 SDValue CallSeqStart = Chain; 5487 5488 // Load the return address and frame pointer so it can be moved somewhere else 5489 // later. 5490 SDValue LROp, FPOp; 5491 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, dl); 5492 5493 // Set up a copy of the stack pointer for use loading and storing any 5494 // arguments that may not fit in the registers available for argument 5495 // passing. 5496 SDValue StackPtr = DAG.getRegister(PPC::R1, MVT::i32); 5497 5498 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 5499 SmallVector<TailCallArgumentInfo, 8> TailCallArguments; 5500 SmallVector<SDValue, 8> MemOpChains; 5501 5502 bool seenFloatArg = false; 5503 // Walk the register/memloc assignments, inserting copies/loads. 5504 // i - Tracks the index into the list of registers allocated for the call 5505 // RealArgIdx - Tracks the index into the list of actual function arguments 5506 // j - Tracks the index into the list of byval arguments 5507 for (unsigned i = 0, RealArgIdx = 0, j = 0, e = ArgLocs.size(); 5508 i != e; 5509 ++i, ++RealArgIdx) { 5510 CCValAssign &VA = ArgLocs[i]; 5511 SDValue Arg = OutVals[RealArgIdx]; 5512 ISD::ArgFlagsTy Flags = Outs[RealArgIdx].Flags; 5513 5514 if (Flags.isByVal()) { 5515 // Argument is an aggregate which is passed by value, thus we need to 5516 // create a copy of it in the local variable space of the current stack 5517 // frame (which is the stack frame of the caller) and pass the address of 5518 // this copy to the callee. 5519 assert((j < ByValArgLocs.size()) && "Index out of bounds!"); 5520 CCValAssign &ByValVA = ByValArgLocs[j++]; 5521 assert((VA.getValNo() == ByValVA.getValNo()) && "ValNo mismatch!"); 5522 5523 // Memory reserved in the local variable space of the callers stack frame. 5524 unsigned LocMemOffset = ByValVA.getLocMemOffset(); 5525 5526 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl); 5527 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(MF.getDataLayout()), 5528 StackPtr, PtrOff); 5529 5530 // Create a copy of the argument in the local area of the current 5531 // stack frame. 5532 SDValue MemcpyCall = 5533 CreateCopyOfByValArgument(Arg, PtrOff, 5534 CallSeqStart.getNode()->getOperand(0), 5535 Flags, DAG, dl); 5536 5537 // This must go outside the CALLSEQ_START..END. 5538 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall, NumBytes, 0, 5539 SDLoc(MemcpyCall)); 5540 DAG.ReplaceAllUsesWith(CallSeqStart.getNode(), 5541 NewCallSeqStart.getNode()); 5542 Chain = CallSeqStart = NewCallSeqStart; 5543 5544 // Pass the address of the aggregate copy on the stack either in a 5545 // physical register or in the parameter list area of the current stack 5546 // frame to the callee. 5547 Arg = PtrOff; 5548 } 5549 5550 // When useCRBits() is true, there can be i1 arguments. 5551 // It is because getRegisterType(MVT::i1) => MVT::i1, 5552 // and for other integer types getRegisterType() => MVT::i32. 5553 // Extend i1 and ensure callee will get i32. 5554 if (Arg.getValueType() == MVT::i1) 5555 Arg = DAG.getNode(Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, 5556 dl, MVT::i32, Arg); 5557 5558 if (VA.isRegLoc()) { 5559 seenFloatArg |= VA.getLocVT().isFloatingPoint(); 5560 // Put argument in a physical register. 5561 if (Subtarget.hasSPE() && Arg.getValueType() == MVT::f64) { 5562 bool IsLE = Subtarget.isLittleEndian(); 5563 SDValue SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg, 5564 DAG.getIntPtrConstant(IsLE ? 0 : 1, dl)); 5565 RegsToPass.push_back(std::make_pair(VA.getLocReg(), SVal.getValue(0))); 5566 SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg, 5567 DAG.getIntPtrConstant(IsLE ? 1 : 0, dl)); 5568 RegsToPass.push_back(std::make_pair(ArgLocs[++i].getLocReg(), 5569 SVal.getValue(0))); 5570 } else 5571 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 5572 } else { 5573 // Put argument in the parameter list area of the current stack frame. 5574 assert(VA.isMemLoc()); 5575 unsigned LocMemOffset = VA.getLocMemOffset(); 5576 5577 if (!isTailCall) { 5578 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl); 5579 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(MF.getDataLayout()), 5580 StackPtr, PtrOff); 5581 5582 MemOpChains.push_back( 5583 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo())); 5584 } else { 5585 // Calculate and remember argument location. 5586 CalculateTailCallArgDest(DAG, MF, false, Arg, SPDiff, LocMemOffset, 5587 TailCallArguments); 5588 } 5589 } 5590 } 5591 5592 if (!MemOpChains.empty()) 5593 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); 5594 5595 // Build a sequence of copy-to-reg nodes chained together with token chain 5596 // and flag operands which copy the outgoing args into the appropriate regs. 5597 SDValue InFlag; 5598 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 5599 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 5600 RegsToPass[i].second, InFlag); 5601 InFlag = Chain.getValue(1); 5602 } 5603 5604 // Set CR bit 6 to true if this is a vararg call with floating args passed in 5605 // registers. 5606 if (isVarArg) { 5607 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue); 5608 SDValue Ops[] = { Chain, InFlag }; 5609 5610 Chain = DAG.getNode(seenFloatArg ? PPCISD::CR6SET : PPCISD::CR6UNSET, 5611 dl, VTs, makeArrayRef(Ops, InFlag.getNode() ? 2 : 1)); 5612 5613 InFlag = Chain.getValue(1); 5614 } 5615 5616 if (isTailCall) 5617 PrepareTailCall(DAG, InFlag, Chain, dl, SPDiff, NumBytes, LROp, FPOp, 5618 TailCallArguments); 5619 5620 return FinishCall(CallConv, dl, isTailCall, isVarArg, isPatchPoint, 5621 /* unused except on PPC64 ELFv1 */ false, DAG, 5622 RegsToPass, InFlag, Chain, CallSeqStart, Callee, SPDiff, 5623 NumBytes, Ins, InVals, CS); 5624 } 5625 5626 // Copy an argument into memory, being careful to do this outside the 5627 // call sequence for the call to which the argument belongs. 5628 SDValue PPCTargetLowering::createMemcpyOutsideCallSeq( 5629 SDValue Arg, SDValue PtrOff, SDValue CallSeqStart, ISD::ArgFlagsTy Flags, 5630 SelectionDAG &DAG, const SDLoc &dl) const { 5631 SDValue MemcpyCall = CreateCopyOfByValArgument(Arg, PtrOff, 5632 CallSeqStart.getNode()->getOperand(0), 5633 Flags, DAG, dl); 5634 // The MEMCPY must go outside the CALLSEQ_START..END. 5635 int64_t FrameSize = CallSeqStart.getConstantOperandVal(1); 5636 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall, FrameSize, 0, 5637 SDLoc(MemcpyCall)); 5638 DAG.ReplaceAllUsesWith(CallSeqStart.getNode(), 5639 NewCallSeqStart.getNode()); 5640 return NewCallSeqStart; 5641 } 5642 5643 SDValue PPCTargetLowering::LowerCall_64SVR4( 5644 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg, 5645 bool isTailCall, bool isPatchPoint, 5646 const SmallVectorImpl<ISD::OutputArg> &Outs, 5647 const SmallVectorImpl<SDValue> &OutVals, 5648 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 5649 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, 5650 ImmutableCallSite CS) const { 5651 bool isELFv2ABI = Subtarget.isELFv2ABI(); 5652 bool isLittleEndian = Subtarget.isLittleEndian(); 5653 unsigned NumOps = Outs.size(); 5654 bool hasNest = false; 5655 bool IsSibCall = false; 5656 5657 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 5658 unsigned PtrByteSize = 8; 5659 5660 MachineFunction &MF = DAG.getMachineFunction(); 5661 5662 if (isTailCall && !getTargetMachine().Options.GuaranteedTailCallOpt) 5663 IsSibCall = true; 5664 5665 // Mark this function as potentially containing a function that contains a 5666 // tail call. As a consequence the frame pointer will be used for dynamicalloc 5667 // and restoring the callers stack pointer in this functions epilog. This is 5668 // done because by tail calling the called function might overwrite the value 5669 // in this function's (MF) stack pointer stack slot 0(SP). 5670 if (getTargetMachine().Options.GuaranteedTailCallOpt && 5671 CallConv == CallingConv::Fast) 5672 MF.getInfo<PPCFunctionInfo>()->setHasFastCall(); 5673 5674 assert(!(CallConv == CallingConv::Fast && isVarArg) && 5675 "fastcc not supported on varargs functions"); 5676 5677 // Count how many bytes are to be pushed on the stack, including the linkage 5678 // area, and parameter passing area. On ELFv1, the linkage area is 48 bytes 5679 // reserved space for [SP][CR][LR][2 x unused][TOC]; on ELFv2, the linkage 5680 // area is 32 bytes reserved space for [SP][CR][LR][TOC]. 5681 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 5682 unsigned NumBytes = LinkageSize; 5683 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; 5684 unsigned &QFPR_idx = FPR_idx; 5685 5686 static const MCPhysReg GPR[] = { 5687 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 5688 PPC::X7, PPC::X8, PPC::X9, PPC::X10, 5689 }; 5690 static const MCPhysReg VR[] = { 5691 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, 5692 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 5693 }; 5694 5695 const unsigned NumGPRs = array_lengthof(GPR); 5696 const unsigned NumFPRs = useSoftFloat() ? 0 : 13; 5697 const unsigned NumVRs = array_lengthof(VR); 5698 const unsigned NumQFPRs = NumFPRs; 5699 5700 // On ELFv2, we can avoid allocating the parameter area if all the arguments 5701 // can be passed to the callee in registers. 5702 // For the fast calling convention, there is another check below. 5703 // Note: We should keep consistent with LowerFormalArguments_64SVR4() 5704 bool HasParameterArea = !isELFv2ABI || isVarArg || CallConv == CallingConv::Fast; 5705 if (!HasParameterArea) { 5706 unsigned ParamAreaSize = NumGPRs * PtrByteSize; 5707 unsigned AvailableFPRs = NumFPRs; 5708 unsigned AvailableVRs = NumVRs; 5709 unsigned NumBytesTmp = NumBytes; 5710 for (unsigned i = 0; i != NumOps; ++i) { 5711 if (Outs[i].Flags.isNest()) continue; 5712 if (CalculateStackSlotUsed(Outs[i].VT, Outs[i].ArgVT, Outs[i].Flags, 5713 PtrByteSize, LinkageSize, ParamAreaSize, 5714 NumBytesTmp, AvailableFPRs, AvailableVRs, 5715 Subtarget.hasQPX())) 5716 HasParameterArea = true; 5717 } 5718 } 5719 5720 // When using the fast calling convention, we don't provide backing for 5721 // arguments that will be in registers. 5722 unsigned NumGPRsUsed = 0, NumFPRsUsed = 0, NumVRsUsed = 0; 5723 5724 // Avoid allocating parameter area for fastcc functions if all the arguments 5725 // can be passed in the registers. 5726 if (CallConv == CallingConv::Fast) 5727 HasParameterArea = false; 5728 5729 // Add up all the space actually used. 5730 for (unsigned i = 0; i != NumOps; ++i) { 5731 ISD::ArgFlagsTy Flags = Outs[i].Flags; 5732 EVT ArgVT = Outs[i].VT; 5733 EVT OrigVT = Outs[i].ArgVT; 5734 5735 if (Flags.isNest()) 5736 continue; 5737 5738 if (CallConv == CallingConv::Fast) { 5739 if (Flags.isByVal()) { 5740 NumGPRsUsed += (Flags.getByValSize()+7)/8; 5741 if (NumGPRsUsed > NumGPRs) 5742 HasParameterArea = true; 5743 } else { 5744 switch (ArgVT.getSimpleVT().SimpleTy) { 5745 default: llvm_unreachable("Unexpected ValueType for argument!"); 5746 case MVT::i1: 5747 case MVT::i32: 5748 case MVT::i64: 5749 if (++NumGPRsUsed <= NumGPRs) 5750 continue; 5751 break; 5752 case MVT::v4i32: 5753 case MVT::v8i16: 5754 case MVT::v16i8: 5755 case MVT::v2f64: 5756 case MVT::v2i64: 5757 case MVT::v1i128: 5758 case MVT::f128: 5759 if (++NumVRsUsed <= NumVRs) 5760 continue; 5761 break; 5762 case MVT::v4f32: 5763 // When using QPX, this is handled like a FP register, otherwise, it 5764 // is an Altivec register. 5765 if (Subtarget.hasQPX()) { 5766 if (++NumFPRsUsed <= NumFPRs) 5767 continue; 5768 } else { 5769 if (++NumVRsUsed <= NumVRs) 5770 continue; 5771 } 5772 break; 5773 case MVT::f32: 5774 case MVT::f64: 5775 case MVT::v4f64: // QPX 5776 case MVT::v4i1: // QPX 5777 if (++NumFPRsUsed <= NumFPRs) 5778 continue; 5779 break; 5780 } 5781 HasParameterArea = true; 5782 } 5783 } 5784 5785 /* Respect alignment of argument on the stack. */ 5786 unsigned Align = 5787 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize); 5788 NumBytes = ((NumBytes + Align - 1) / Align) * Align; 5789 5790 NumBytes += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize); 5791 if (Flags.isInConsecutiveRegsLast()) 5792 NumBytes = ((NumBytes + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 5793 } 5794 5795 unsigned NumBytesActuallyUsed = NumBytes; 5796 5797 // In the old ELFv1 ABI, 5798 // the prolog code of the callee may store up to 8 GPR argument registers to 5799 // the stack, allowing va_start to index over them in memory if its varargs. 5800 // Because we cannot tell if this is needed on the caller side, we have to 5801 // conservatively assume that it is needed. As such, make sure we have at 5802 // least enough stack space for the caller to store the 8 GPRs. 5803 // In the ELFv2 ABI, we allocate the parameter area iff a callee 5804 // really requires memory operands, e.g. a vararg function. 5805 if (HasParameterArea) 5806 NumBytes = std::max(NumBytes, LinkageSize + 8 * PtrByteSize); 5807 else 5808 NumBytes = LinkageSize; 5809 5810 // Tail call needs the stack to be aligned. 5811 if (getTargetMachine().Options.GuaranteedTailCallOpt && 5812 CallConv == CallingConv::Fast) 5813 NumBytes = EnsureStackAlignment(Subtarget.getFrameLowering(), NumBytes); 5814 5815 int SPDiff = 0; 5816 5817 // Calculate by how many bytes the stack has to be adjusted in case of tail 5818 // call optimization. 5819 if (!IsSibCall) 5820 SPDiff = CalculateTailCallSPDiff(DAG, isTailCall, NumBytes); 5821 5822 // To protect arguments on the stack from being clobbered in a tail call, 5823 // force all the loads to happen before doing any other lowering. 5824 if (isTailCall) 5825 Chain = DAG.getStackArgumentTokenFactor(Chain); 5826 5827 // Adjust the stack pointer for the new arguments... 5828 // These operations are automatically eliminated by the prolog/epilog pass 5829 if (!IsSibCall) 5830 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); 5831 SDValue CallSeqStart = Chain; 5832 5833 // Load the return address and frame pointer so it can be move somewhere else 5834 // later. 5835 SDValue LROp, FPOp; 5836 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, dl); 5837 5838 // Set up a copy of the stack pointer for use loading and storing any 5839 // arguments that may not fit in the registers available for argument 5840 // passing. 5841 SDValue StackPtr = DAG.getRegister(PPC::X1, MVT::i64); 5842 5843 // Figure out which arguments are going to go in registers, and which in 5844 // memory. Also, if this is a vararg function, floating point operations 5845 // must be stored to our stack, and loaded into integer regs as well, if 5846 // any integer regs are available for argument passing. 5847 unsigned ArgOffset = LinkageSize; 5848 5849 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 5850 SmallVector<TailCallArgumentInfo, 8> TailCallArguments; 5851 5852 SmallVector<SDValue, 8> MemOpChains; 5853 for (unsigned i = 0; i != NumOps; ++i) { 5854 SDValue Arg = OutVals[i]; 5855 ISD::ArgFlagsTy Flags = Outs[i].Flags; 5856 EVT ArgVT = Outs[i].VT; 5857 EVT OrigVT = Outs[i].ArgVT; 5858 5859 // PtrOff will be used to store the current argument to the stack if a 5860 // register cannot be found for it. 5861 SDValue PtrOff; 5862 5863 // We re-align the argument offset for each argument, except when using the 5864 // fast calling convention, when we need to make sure we do that only when 5865 // we'll actually use a stack slot. 5866 auto ComputePtrOff = [&]() { 5867 /* Respect alignment of argument on the stack. */ 5868 unsigned Align = 5869 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize); 5870 ArgOffset = ((ArgOffset + Align - 1) / Align) * Align; 5871 5872 PtrOff = DAG.getConstant(ArgOffset, dl, StackPtr.getValueType()); 5873 5874 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); 5875 }; 5876 5877 if (CallConv != CallingConv::Fast) { 5878 ComputePtrOff(); 5879 5880 /* Compute GPR index associated with argument offset. */ 5881 GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize; 5882 GPR_idx = std::min(GPR_idx, NumGPRs); 5883 } 5884 5885 // Promote integers to 64-bit values. 5886 if (Arg.getValueType() == MVT::i32 || Arg.getValueType() == MVT::i1) { 5887 // FIXME: Should this use ANY_EXTEND if neither sext nor zext? 5888 unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 5889 Arg = DAG.getNode(ExtOp, dl, MVT::i64, Arg); 5890 } 5891 5892 // FIXME memcpy is used way more than necessary. Correctness first. 5893 // Note: "by value" is code for passing a structure by value, not 5894 // basic types. 5895 if (Flags.isByVal()) { 5896 // Note: Size includes alignment padding, so 5897 // struct x { short a; char b; } 5898 // will have Size = 4. With #pragma pack(1), it will have Size = 3. 5899 // These are the proper values we need for right-justifying the 5900 // aggregate in a parameter register. 5901 unsigned Size = Flags.getByValSize(); 5902 5903 // An empty aggregate parameter takes up no storage and no 5904 // registers. 5905 if (Size == 0) 5906 continue; 5907 5908 if (CallConv == CallingConv::Fast) 5909 ComputePtrOff(); 5910 5911 // All aggregates smaller than 8 bytes must be passed right-justified. 5912 if (Size==1 || Size==2 || Size==4) { 5913 EVT VT = (Size==1) ? MVT::i8 : ((Size==2) ? MVT::i16 : MVT::i32); 5914 if (GPR_idx != NumGPRs) { 5915 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg, 5916 MachinePointerInfo(), VT); 5917 MemOpChains.push_back(Load.getValue(1)); 5918 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 5919 5920 ArgOffset += PtrByteSize; 5921 continue; 5922 } 5923 } 5924 5925 if (GPR_idx == NumGPRs && Size < 8) { 5926 SDValue AddPtr = PtrOff; 5927 if (!isLittleEndian) { 5928 SDValue Const = DAG.getConstant(PtrByteSize - Size, dl, 5929 PtrOff.getValueType()); 5930 AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const); 5931 } 5932 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr, 5933 CallSeqStart, 5934 Flags, DAG, dl); 5935 ArgOffset += PtrByteSize; 5936 continue; 5937 } 5938 // Copy entire object into memory. There are cases where gcc-generated 5939 // code assumes it is there, even if it could be put entirely into 5940 // registers. (This is not what the doc says.) 5941 5942 // FIXME: The above statement is likely due to a misunderstanding of the 5943 // documents. All arguments must be copied into the parameter area BY 5944 // THE CALLEE in the event that the callee takes the address of any 5945 // formal argument. That has not yet been implemented. However, it is 5946 // reasonable to use the stack area as a staging area for the register 5947 // load. 5948 5949 // Skip this for small aggregates, as we will use the same slot for a 5950 // right-justified copy, below. 5951 if (Size >= 8) 5952 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff, 5953 CallSeqStart, 5954 Flags, DAG, dl); 5955 5956 // When a register is available, pass a small aggregate right-justified. 5957 if (Size < 8 && GPR_idx != NumGPRs) { 5958 // The easiest way to get this right-justified in a register 5959 // is to copy the structure into the rightmost portion of a 5960 // local variable slot, then load the whole slot into the 5961 // register. 5962 // FIXME: The memcpy seems to produce pretty awful code for 5963 // small aggregates, particularly for packed ones. 5964 // FIXME: It would be preferable to use the slot in the 5965 // parameter save area instead of a new local variable. 5966 SDValue AddPtr = PtrOff; 5967 if (!isLittleEndian) { 5968 SDValue Const = DAG.getConstant(8 - Size, dl, PtrOff.getValueType()); 5969 AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const); 5970 } 5971 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr, 5972 CallSeqStart, 5973 Flags, DAG, dl); 5974 5975 // Load the slot into the register. 5976 SDValue Load = 5977 DAG.getLoad(PtrVT, dl, Chain, PtrOff, MachinePointerInfo()); 5978 MemOpChains.push_back(Load.getValue(1)); 5979 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 5980 5981 // Done with this argument. 5982 ArgOffset += PtrByteSize; 5983 continue; 5984 } 5985 5986 // For aggregates larger than PtrByteSize, copy the pieces of the 5987 // object that fit into registers from the parameter save area. 5988 for (unsigned j=0; j<Size; j+=PtrByteSize) { 5989 SDValue Const = DAG.getConstant(j, dl, PtrOff.getValueType()); 5990 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const); 5991 if (GPR_idx != NumGPRs) { 5992 SDValue Load = 5993 DAG.getLoad(PtrVT, dl, Chain, AddArg, MachinePointerInfo()); 5994 MemOpChains.push_back(Load.getValue(1)); 5995 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 5996 ArgOffset += PtrByteSize; 5997 } else { 5998 ArgOffset += ((Size - j + PtrByteSize-1)/PtrByteSize)*PtrByteSize; 5999 break; 6000 } 6001 } 6002 continue; 6003 } 6004 6005 switch (Arg.getSimpleValueType().SimpleTy) { 6006 default: llvm_unreachable("Unexpected ValueType for argument!"); 6007 case MVT::i1: 6008 case MVT::i32: 6009 case MVT::i64: 6010 if (Flags.isNest()) { 6011 // The 'nest' parameter, if any, is passed in R11. 6012 RegsToPass.push_back(std::make_pair(PPC::X11, Arg)); 6013 hasNest = true; 6014 break; 6015 } 6016 6017 // These can be scalar arguments or elements of an integer array type 6018 // passed directly. Clang may use those instead of "byval" aggregate 6019 // types to avoid forcing arguments to memory unnecessarily. 6020 if (GPR_idx != NumGPRs) { 6021 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Arg)); 6022 } else { 6023 if (CallConv == CallingConv::Fast) 6024 ComputePtrOff(); 6025 6026 assert(HasParameterArea && 6027 "Parameter area must exist to pass an argument in memory."); 6028 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 6029 true, isTailCall, false, MemOpChains, 6030 TailCallArguments, dl); 6031 if (CallConv == CallingConv::Fast) 6032 ArgOffset += PtrByteSize; 6033 } 6034 if (CallConv != CallingConv::Fast) 6035 ArgOffset += PtrByteSize; 6036 break; 6037 case MVT::f32: 6038 case MVT::f64: { 6039 // These can be scalar arguments or elements of a float array type 6040 // passed directly. The latter are used to implement ELFv2 homogenous 6041 // float aggregates. 6042 6043 // Named arguments go into FPRs first, and once they overflow, the 6044 // remaining arguments go into GPRs and then the parameter save area. 6045 // Unnamed arguments for vararg functions always go to GPRs and 6046 // then the parameter save area. For now, put all arguments to vararg 6047 // routines always in both locations (FPR *and* GPR or stack slot). 6048 bool NeedGPROrStack = isVarArg || FPR_idx == NumFPRs; 6049 bool NeededLoad = false; 6050 6051 // First load the argument into the next available FPR. 6052 if (FPR_idx != NumFPRs) 6053 RegsToPass.push_back(std::make_pair(FPR[FPR_idx++], Arg)); 6054 6055 // Next, load the argument into GPR or stack slot if needed. 6056 if (!NeedGPROrStack) 6057 ; 6058 else if (GPR_idx != NumGPRs && CallConv != CallingConv::Fast) { 6059 // FIXME: We may want to re-enable this for CallingConv::Fast on the P8 6060 // once we support fp <-> gpr moves. 6061 6062 // In the non-vararg case, this can only ever happen in the 6063 // presence of f32 array types, since otherwise we never run 6064 // out of FPRs before running out of GPRs. 6065 SDValue ArgVal; 6066 6067 // Double values are always passed in a single GPR. 6068 if (Arg.getValueType() != MVT::f32) { 6069 ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i64, Arg); 6070 6071 // Non-array float values are extended and passed in a GPR. 6072 } else if (!Flags.isInConsecutiveRegs()) { 6073 ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg); 6074 ArgVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, ArgVal); 6075 6076 // If we have an array of floats, we collect every odd element 6077 // together with its predecessor into one GPR. 6078 } else if (ArgOffset % PtrByteSize != 0) { 6079 SDValue Lo, Hi; 6080 Lo = DAG.getNode(ISD::BITCAST, dl, MVT::i32, OutVals[i - 1]); 6081 Hi = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg); 6082 if (!isLittleEndian) 6083 std::swap(Lo, Hi); 6084 ArgVal = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi); 6085 6086 // The final element, if even, goes into the first half of a GPR. 6087 } else if (Flags.isInConsecutiveRegsLast()) { 6088 ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg); 6089 ArgVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, ArgVal); 6090 if (!isLittleEndian) 6091 ArgVal = DAG.getNode(ISD::SHL, dl, MVT::i64, ArgVal, 6092 DAG.getConstant(32, dl, MVT::i32)); 6093 6094 // Non-final even elements are skipped; they will be handled 6095 // together the with subsequent argument on the next go-around. 6096 } else 6097 ArgVal = SDValue(); 6098 6099 if (ArgVal.getNode()) 6100 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], ArgVal)); 6101 } else { 6102 if (CallConv == CallingConv::Fast) 6103 ComputePtrOff(); 6104 6105 // Single-precision floating-point values are mapped to the 6106 // second (rightmost) word of the stack doubleword. 6107 if (Arg.getValueType() == MVT::f32 && 6108 !isLittleEndian && !Flags.isInConsecutiveRegs()) { 6109 SDValue ConstFour = DAG.getConstant(4, dl, PtrOff.getValueType()); 6110 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, ConstFour); 6111 } 6112 6113 assert(HasParameterArea && 6114 "Parameter area must exist to pass an argument in memory."); 6115 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 6116 true, isTailCall, false, MemOpChains, 6117 TailCallArguments, dl); 6118 6119 NeededLoad = true; 6120 } 6121 // When passing an array of floats, the array occupies consecutive 6122 // space in the argument area; only round up to the next doubleword 6123 // at the end of the array. Otherwise, each float takes 8 bytes. 6124 if (CallConv != CallingConv::Fast || NeededLoad) { 6125 ArgOffset += (Arg.getValueType() == MVT::f32 && 6126 Flags.isInConsecutiveRegs()) ? 4 : 8; 6127 if (Flags.isInConsecutiveRegsLast()) 6128 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 6129 } 6130 break; 6131 } 6132 case MVT::v4f32: 6133 case MVT::v4i32: 6134 case MVT::v8i16: 6135 case MVT::v16i8: 6136 case MVT::v2f64: 6137 case MVT::v2i64: 6138 case MVT::v1i128: 6139 case MVT::f128: 6140 if (!Subtarget.hasQPX()) { 6141 // These can be scalar arguments or elements of a vector array type 6142 // passed directly. The latter are used to implement ELFv2 homogenous 6143 // vector aggregates. 6144 6145 // For a varargs call, named arguments go into VRs or on the stack as 6146 // usual; unnamed arguments always go to the stack or the corresponding 6147 // GPRs when within range. For now, we always put the value in both 6148 // locations (or even all three). 6149 if (isVarArg) { 6150 assert(HasParameterArea && 6151 "Parameter area must exist if we have a varargs call."); 6152 // We could elide this store in the case where the object fits 6153 // entirely in R registers. Maybe later. 6154 SDValue Store = 6155 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()); 6156 MemOpChains.push_back(Store); 6157 if (VR_idx != NumVRs) { 6158 SDValue Load = 6159 DAG.getLoad(MVT::v4f32, dl, Store, PtrOff, MachinePointerInfo()); 6160 MemOpChains.push_back(Load.getValue(1)); 6161 RegsToPass.push_back(std::make_pair(VR[VR_idx++], Load)); 6162 } 6163 ArgOffset += 16; 6164 for (unsigned i=0; i<16; i+=PtrByteSize) { 6165 if (GPR_idx == NumGPRs) 6166 break; 6167 SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, 6168 DAG.getConstant(i, dl, PtrVT)); 6169 SDValue Load = 6170 DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo()); 6171 MemOpChains.push_back(Load.getValue(1)); 6172 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 6173 } 6174 break; 6175 } 6176 6177 // Non-varargs Altivec params go into VRs or on the stack. 6178 if (VR_idx != NumVRs) { 6179 RegsToPass.push_back(std::make_pair(VR[VR_idx++], Arg)); 6180 } else { 6181 if (CallConv == CallingConv::Fast) 6182 ComputePtrOff(); 6183 6184 assert(HasParameterArea && 6185 "Parameter area must exist to pass an argument in memory."); 6186 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 6187 true, isTailCall, true, MemOpChains, 6188 TailCallArguments, dl); 6189 if (CallConv == CallingConv::Fast) 6190 ArgOffset += 16; 6191 } 6192 6193 if (CallConv != CallingConv::Fast) 6194 ArgOffset += 16; 6195 break; 6196 } // not QPX 6197 6198 assert(Arg.getValueType().getSimpleVT().SimpleTy == MVT::v4f32 && 6199 "Invalid QPX parameter type"); 6200 6201 LLVM_FALLTHROUGH; 6202 case MVT::v4f64: 6203 case MVT::v4i1: { 6204 bool IsF32 = Arg.getValueType().getSimpleVT().SimpleTy == MVT::v4f32; 6205 if (isVarArg) { 6206 assert(HasParameterArea && 6207 "Parameter area must exist if we have a varargs call."); 6208 // We could elide this store in the case where the object fits 6209 // entirely in R registers. Maybe later. 6210 SDValue Store = 6211 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()); 6212 MemOpChains.push_back(Store); 6213 if (QFPR_idx != NumQFPRs) { 6214 SDValue Load = DAG.getLoad(IsF32 ? MVT::v4f32 : MVT::v4f64, dl, Store, 6215 PtrOff, MachinePointerInfo()); 6216 MemOpChains.push_back(Load.getValue(1)); 6217 RegsToPass.push_back(std::make_pair(QFPR[QFPR_idx++], Load)); 6218 } 6219 ArgOffset += (IsF32 ? 16 : 32); 6220 for (unsigned i = 0; i < (IsF32 ? 16U : 32U); i += PtrByteSize) { 6221 if (GPR_idx == NumGPRs) 6222 break; 6223 SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, 6224 DAG.getConstant(i, dl, PtrVT)); 6225 SDValue Load = 6226 DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo()); 6227 MemOpChains.push_back(Load.getValue(1)); 6228 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 6229 } 6230 break; 6231 } 6232 6233 // Non-varargs QPX params go into registers or on the stack. 6234 if (QFPR_idx != NumQFPRs) { 6235 RegsToPass.push_back(std::make_pair(QFPR[QFPR_idx++], Arg)); 6236 } else { 6237 if (CallConv == CallingConv::Fast) 6238 ComputePtrOff(); 6239 6240 assert(HasParameterArea && 6241 "Parameter area must exist to pass an argument in memory."); 6242 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 6243 true, isTailCall, true, MemOpChains, 6244 TailCallArguments, dl); 6245 if (CallConv == CallingConv::Fast) 6246 ArgOffset += (IsF32 ? 16 : 32); 6247 } 6248 6249 if (CallConv != CallingConv::Fast) 6250 ArgOffset += (IsF32 ? 16 : 32); 6251 break; 6252 } 6253 } 6254 } 6255 6256 assert((!HasParameterArea || NumBytesActuallyUsed == ArgOffset) && 6257 "mismatch in size of parameter area"); 6258 (void)NumBytesActuallyUsed; 6259 6260 if (!MemOpChains.empty()) 6261 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); 6262 6263 // Check if this is an indirect call (MTCTR/BCTRL). 6264 // See PrepareCall() for more information about calls through function 6265 // pointers in the 64-bit SVR4 ABI. 6266 if (!isTailCall && !isPatchPoint && 6267 !isFunctionGlobalAddress(Callee) && 6268 !isa<ExternalSymbolSDNode>(Callee)) { 6269 // Load r2 into a virtual register and store it to the TOC save area. 6270 setUsesTOCBasePtr(DAG); 6271 SDValue Val = DAG.getCopyFromReg(Chain, dl, PPC::X2, MVT::i64); 6272 // TOC save area offset. 6273 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset(); 6274 SDValue PtrOff = DAG.getIntPtrConstant(TOCSaveOffset, dl); 6275 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); 6276 Chain = DAG.getStore( 6277 Val.getValue(1), dl, Val, AddPtr, 6278 MachinePointerInfo::getStack(DAG.getMachineFunction(), TOCSaveOffset)); 6279 // In the ELFv2 ABI, R12 must contain the address of an indirect callee. 6280 // This does not mean the MTCTR instruction must use R12; it's easier 6281 // to model this as an extra parameter, so do that. 6282 if (isELFv2ABI && !isPatchPoint) 6283 RegsToPass.push_back(std::make_pair((unsigned)PPC::X12, Callee)); 6284 } 6285 6286 // Build a sequence of copy-to-reg nodes chained together with token chain 6287 // and flag operands which copy the outgoing args into the appropriate regs. 6288 SDValue InFlag; 6289 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 6290 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 6291 RegsToPass[i].second, InFlag); 6292 InFlag = Chain.getValue(1); 6293 } 6294 6295 if (isTailCall && !IsSibCall) 6296 PrepareTailCall(DAG, InFlag, Chain, dl, SPDiff, NumBytes, LROp, FPOp, 6297 TailCallArguments); 6298 6299 return FinishCall(CallConv, dl, isTailCall, isVarArg, isPatchPoint, hasNest, 6300 DAG, RegsToPass, InFlag, Chain, CallSeqStart, Callee, 6301 SPDiff, NumBytes, Ins, InVals, CS); 6302 } 6303 6304 SDValue PPCTargetLowering::LowerCall_Darwin( 6305 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg, 6306 bool isTailCall, bool isPatchPoint, 6307 const SmallVectorImpl<ISD::OutputArg> &Outs, 6308 const SmallVectorImpl<SDValue> &OutVals, 6309 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 6310 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, 6311 ImmutableCallSite CS) const { 6312 unsigned NumOps = Outs.size(); 6313 6314 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 6315 bool isPPC64 = PtrVT == MVT::i64; 6316 unsigned PtrByteSize = isPPC64 ? 8 : 4; 6317 6318 MachineFunction &MF = DAG.getMachineFunction(); 6319 6320 // Mark this function as potentially containing a function that contains a 6321 // tail call. As a consequence the frame pointer will be used for dynamicalloc 6322 // and restoring the callers stack pointer in this functions epilog. This is 6323 // done because by tail calling the called function might overwrite the value 6324 // in this function's (MF) stack pointer stack slot 0(SP). 6325 if (getTargetMachine().Options.GuaranteedTailCallOpt && 6326 CallConv == CallingConv::Fast) 6327 MF.getInfo<PPCFunctionInfo>()->setHasFastCall(); 6328 6329 // Count how many bytes are to be pushed on the stack, including the linkage 6330 // area, and parameter passing area. We start with 24/48 bytes, which is 6331 // prereserved space for [SP][CR][LR][3 x unused]. 6332 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 6333 unsigned NumBytes = LinkageSize; 6334 6335 // Add up all the space actually used. 6336 // In 32-bit non-varargs calls, Altivec parameters all go at the end; usually 6337 // they all go in registers, but we must reserve stack space for them for 6338 // possible use by the caller. In varargs or 64-bit calls, parameters are 6339 // assigned stack space in order, with padding so Altivec parameters are 6340 // 16-byte aligned. 6341 unsigned nAltivecParamsAtEnd = 0; 6342 for (unsigned i = 0; i != NumOps; ++i) { 6343 ISD::ArgFlagsTy Flags = Outs[i].Flags; 6344 EVT ArgVT = Outs[i].VT; 6345 // Varargs Altivec parameters are padded to a 16 byte boundary. 6346 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 || 6347 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 || 6348 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64) { 6349 if (!isVarArg && !isPPC64) { 6350 // Non-varargs Altivec parameters go after all the non-Altivec 6351 // parameters; handle those later so we know how much padding we need. 6352 nAltivecParamsAtEnd++; 6353 continue; 6354 } 6355 // Varargs and 64-bit Altivec parameters are padded to 16 byte boundary. 6356 NumBytes = ((NumBytes+15)/16)*16; 6357 } 6358 NumBytes += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize); 6359 } 6360 6361 // Allow for Altivec parameters at the end, if needed. 6362 if (nAltivecParamsAtEnd) { 6363 NumBytes = ((NumBytes+15)/16)*16; 6364 NumBytes += 16*nAltivecParamsAtEnd; 6365 } 6366 6367 // The prolog code of the callee may store up to 8 GPR argument registers to 6368 // the stack, allowing va_start to index over them in memory if its varargs. 6369 // Because we cannot tell if this is needed on the caller side, we have to 6370 // conservatively assume that it is needed. As such, make sure we have at 6371 // least enough stack space for the caller to store the 8 GPRs. 6372 NumBytes = std::max(NumBytes, LinkageSize + 8 * PtrByteSize); 6373 6374 // Tail call needs the stack to be aligned. 6375 if (getTargetMachine().Options.GuaranteedTailCallOpt && 6376 CallConv == CallingConv::Fast) 6377 NumBytes = EnsureStackAlignment(Subtarget.getFrameLowering(), NumBytes); 6378 6379 // Calculate by how many bytes the stack has to be adjusted in case of tail 6380 // call optimization. 6381 int SPDiff = CalculateTailCallSPDiff(DAG, isTailCall, NumBytes); 6382 6383 // To protect arguments on the stack from being clobbered in a tail call, 6384 // force all the loads to happen before doing any other lowering. 6385 if (isTailCall) 6386 Chain = DAG.getStackArgumentTokenFactor(Chain); 6387 6388 // Adjust the stack pointer for the new arguments... 6389 // These operations are automatically eliminated by the prolog/epilog pass 6390 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); 6391 SDValue CallSeqStart = Chain; 6392 6393 // Load the return address and frame pointer so it can be move somewhere else 6394 // later. 6395 SDValue LROp, FPOp; 6396 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, dl); 6397 6398 // Set up a copy of the stack pointer for use loading and storing any 6399 // arguments that may not fit in the registers available for argument 6400 // passing. 6401 SDValue StackPtr; 6402 if (isPPC64) 6403 StackPtr = DAG.getRegister(PPC::X1, MVT::i64); 6404 else 6405 StackPtr = DAG.getRegister(PPC::R1, MVT::i32); 6406 6407 // Figure out which arguments are going to go in registers, and which in 6408 // memory. Also, if this is a vararg function, floating point operations 6409 // must be stored to our stack, and loaded into integer regs as well, if 6410 // any integer regs are available for argument passing. 6411 unsigned ArgOffset = LinkageSize; 6412 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; 6413 6414 static const MCPhysReg GPR_32[] = { // 32-bit registers. 6415 PPC::R3, PPC::R4, PPC::R5, PPC::R6, 6416 PPC::R7, PPC::R8, PPC::R9, PPC::R10, 6417 }; 6418 static const MCPhysReg GPR_64[] = { // 64-bit registers. 6419 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 6420 PPC::X7, PPC::X8, PPC::X9, PPC::X10, 6421 }; 6422 static const MCPhysReg VR[] = { 6423 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, 6424 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 6425 }; 6426 const unsigned NumGPRs = array_lengthof(GPR_32); 6427 const unsigned NumFPRs = 13; 6428 const unsigned NumVRs = array_lengthof(VR); 6429 6430 const MCPhysReg *GPR = isPPC64 ? GPR_64 : GPR_32; 6431 6432 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 6433 SmallVector<TailCallArgumentInfo, 8> TailCallArguments; 6434 6435 SmallVector<SDValue, 8> MemOpChains; 6436 for (unsigned i = 0; i != NumOps; ++i) { 6437 SDValue Arg = OutVals[i]; 6438 ISD::ArgFlagsTy Flags = Outs[i].Flags; 6439 6440 // PtrOff will be used to store the current argument to the stack if a 6441 // register cannot be found for it. 6442 SDValue PtrOff; 6443 6444 PtrOff = DAG.getConstant(ArgOffset, dl, StackPtr.getValueType()); 6445 6446 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); 6447 6448 // On PPC64, promote integers to 64-bit values. 6449 if (isPPC64 && Arg.getValueType() == MVT::i32) { 6450 // FIXME: Should this use ANY_EXTEND if neither sext nor zext? 6451 unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 6452 Arg = DAG.getNode(ExtOp, dl, MVT::i64, Arg); 6453 } 6454 6455 // FIXME memcpy is used way more than necessary. Correctness first. 6456 // Note: "by value" is code for passing a structure by value, not 6457 // basic types. 6458 if (Flags.isByVal()) { 6459 unsigned Size = Flags.getByValSize(); 6460 // Very small objects are passed right-justified. Everything else is 6461 // passed left-justified. 6462 if (Size==1 || Size==2) { 6463 EVT VT = (Size==1) ? MVT::i8 : MVT::i16; 6464 if (GPR_idx != NumGPRs) { 6465 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg, 6466 MachinePointerInfo(), VT); 6467 MemOpChains.push_back(Load.getValue(1)); 6468 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 6469 6470 ArgOffset += PtrByteSize; 6471 } else { 6472 SDValue Const = DAG.getConstant(PtrByteSize - Size, dl, 6473 PtrOff.getValueType()); 6474 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const); 6475 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr, 6476 CallSeqStart, 6477 Flags, DAG, dl); 6478 ArgOffset += PtrByteSize; 6479 } 6480 continue; 6481 } 6482 // Copy entire object into memory. There are cases where gcc-generated 6483 // code assumes it is there, even if it could be put entirely into 6484 // registers. (This is not what the doc says.) 6485 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff, 6486 CallSeqStart, 6487 Flags, DAG, dl); 6488 6489 // For small aggregates (Darwin only) and aggregates >= PtrByteSize, 6490 // copy the pieces of the object that fit into registers from the 6491 // parameter save area. 6492 for (unsigned j=0; j<Size; j+=PtrByteSize) { 6493 SDValue Const = DAG.getConstant(j, dl, PtrOff.getValueType()); 6494 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const); 6495 if (GPR_idx != NumGPRs) { 6496 SDValue Load = 6497 DAG.getLoad(PtrVT, dl, Chain, AddArg, MachinePointerInfo()); 6498 MemOpChains.push_back(Load.getValue(1)); 6499 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 6500 ArgOffset += PtrByteSize; 6501 } else { 6502 ArgOffset += ((Size - j + PtrByteSize-1)/PtrByteSize)*PtrByteSize; 6503 break; 6504 } 6505 } 6506 continue; 6507 } 6508 6509 switch (Arg.getSimpleValueType().SimpleTy) { 6510 default: llvm_unreachable("Unexpected ValueType for argument!"); 6511 case MVT::i1: 6512 case MVT::i32: 6513 case MVT::i64: 6514 if (GPR_idx != NumGPRs) { 6515 if (Arg.getValueType() == MVT::i1) 6516 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, PtrVT, Arg); 6517 6518 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Arg)); 6519 } else { 6520 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 6521 isPPC64, isTailCall, false, MemOpChains, 6522 TailCallArguments, dl); 6523 } 6524 ArgOffset += PtrByteSize; 6525 break; 6526 case MVT::f32: 6527 case MVT::f64: 6528 if (FPR_idx != NumFPRs) { 6529 RegsToPass.push_back(std::make_pair(FPR[FPR_idx++], Arg)); 6530 6531 if (isVarArg) { 6532 SDValue Store = 6533 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()); 6534 MemOpChains.push_back(Store); 6535 6536 // Float varargs are always shadowed in available integer registers 6537 if (GPR_idx != NumGPRs) { 6538 SDValue Load = 6539 DAG.getLoad(PtrVT, dl, Store, PtrOff, MachinePointerInfo()); 6540 MemOpChains.push_back(Load.getValue(1)); 6541 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 6542 } 6543 if (GPR_idx != NumGPRs && Arg.getValueType() == MVT::f64 && !isPPC64){ 6544 SDValue ConstFour = DAG.getConstant(4, dl, PtrOff.getValueType()); 6545 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, ConstFour); 6546 SDValue Load = 6547 DAG.getLoad(PtrVT, dl, Store, PtrOff, MachinePointerInfo()); 6548 MemOpChains.push_back(Load.getValue(1)); 6549 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 6550 } 6551 } else { 6552 // If we have any FPRs remaining, we may also have GPRs remaining. 6553 // Args passed in FPRs consume either 1 (f32) or 2 (f64) available 6554 // GPRs. 6555 if (GPR_idx != NumGPRs) 6556 ++GPR_idx; 6557 if (GPR_idx != NumGPRs && Arg.getValueType() == MVT::f64 && 6558 !isPPC64) // PPC64 has 64-bit GPR's obviously :) 6559 ++GPR_idx; 6560 } 6561 } else 6562 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 6563 isPPC64, isTailCall, false, MemOpChains, 6564 TailCallArguments, dl); 6565 if (isPPC64) 6566 ArgOffset += 8; 6567 else 6568 ArgOffset += Arg.getValueType() == MVT::f32 ? 4 : 8; 6569 break; 6570 case MVT::v4f32: 6571 case MVT::v4i32: 6572 case MVT::v8i16: 6573 case MVT::v16i8: 6574 if (isVarArg) { 6575 // These go aligned on the stack, or in the corresponding R registers 6576 // when within range. The Darwin PPC ABI doc claims they also go in 6577 // V registers; in fact gcc does this only for arguments that are 6578 // prototyped, not for those that match the ... We do it for all 6579 // arguments, seems to work. 6580 while (ArgOffset % 16 !=0) { 6581 ArgOffset += PtrByteSize; 6582 if (GPR_idx != NumGPRs) 6583 GPR_idx++; 6584 } 6585 // We could elide this store in the case where the object fits 6586 // entirely in R registers. Maybe later. 6587 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, 6588 DAG.getConstant(ArgOffset, dl, PtrVT)); 6589 SDValue Store = 6590 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()); 6591 MemOpChains.push_back(Store); 6592 if (VR_idx != NumVRs) { 6593 SDValue Load = 6594 DAG.getLoad(MVT::v4f32, dl, Store, PtrOff, MachinePointerInfo()); 6595 MemOpChains.push_back(Load.getValue(1)); 6596 RegsToPass.push_back(std::make_pair(VR[VR_idx++], Load)); 6597 } 6598 ArgOffset += 16; 6599 for (unsigned i=0; i<16; i+=PtrByteSize) { 6600 if (GPR_idx == NumGPRs) 6601 break; 6602 SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, 6603 DAG.getConstant(i, dl, PtrVT)); 6604 SDValue Load = 6605 DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo()); 6606 MemOpChains.push_back(Load.getValue(1)); 6607 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 6608 } 6609 break; 6610 } 6611 6612 // Non-varargs Altivec params generally go in registers, but have 6613 // stack space allocated at the end. 6614 if (VR_idx != NumVRs) { 6615 // Doesn't have GPR space allocated. 6616 RegsToPass.push_back(std::make_pair(VR[VR_idx++], Arg)); 6617 } else if (nAltivecParamsAtEnd==0) { 6618 // We are emitting Altivec params in order. 6619 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 6620 isPPC64, isTailCall, true, MemOpChains, 6621 TailCallArguments, dl); 6622 ArgOffset += 16; 6623 } 6624 break; 6625 } 6626 } 6627 // If all Altivec parameters fit in registers, as they usually do, 6628 // they get stack space following the non-Altivec parameters. We 6629 // don't track this here because nobody below needs it. 6630 // If there are more Altivec parameters than fit in registers emit 6631 // the stores here. 6632 if (!isVarArg && nAltivecParamsAtEnd > NumVRs) { 6633 unsigned j = 0; 6634 // Offset is aligned; skip 1st 12 params which go in V registers. 6635 ArgOffset = ((ArgOffset+15)/16)*16; 6636 ArgOffset += 12*16; 6637 for (unsigned i = 0; i != NumOps; ++i) { 6638 SDValue Arg = OutVals[i]; 6639 EVT ArgType = Outs[i].VT; 6640 if (ArgType==MVT::v4f32 || ArgType==MVT::v4i32 || 6641 ArgType==MVT::v8i16 || ArgType==MVT::v16i8) { 6642 if (++j > NumVRs) { 6643 SDValue PtrOff; 6644 // We are emitting Altivec params in order. 6645 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 6646 isPPC64, isTailCall, true, MemOpChains, 6647 TailCallArguments, dl); 6648 ArgOffset += 16; 6649 } 6650 } 6651 } 6652 } 6653 6654 if (!MemOpChains.empty()) 6655 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); 6656 6657 // On Darwin, R12 must contain the address of an indirect callee. This does 6658 // not mean the MTCTR instruction must use R12; it's easier to model this as 6659 // an extra parameter, so do that. 6660 if (!isTailCall && 6661 !isFunctionGlobalAddress(Callee) && 6662 !isa<ExternalSymbolSDNode>(Callee) && 6663 !isBLACompatibleAddress(Callee, DAG)) 6664 RegsToPass.push_back(std::make_pair((unsigned)(isPPC64 ? PPC::X12 : 6665 PPC::R12), Callee)); 6666 6667 // Build a sequence of copy-to-reg nodes chained together with token chain 6668 // and flag operands which copy the outgoing args into the appropriate regs. 6669 SDValue InFlag; 6670 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 6671 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 6672 RegsToPass[i].second, InFlag); 6673 InFlag = Chain.getValue(1); 6674 } 6675 6676 if (isTailCall) 6677 PrepareTailCall(DAG, InFlag, Chain, dl, SPDiff, NumBytes, LROp, FPOp, 6678 TailCallArguments); 6679 6680 return FinishCall(CallConv, dl, isTailCall, isVarArg, isPatchPoint, 6681 /* unused except on PPC64 ELFv1 */ false, DAG, 6682 RegsToPass, InFlag, Chain, CallSeqStart, Callee, SPDiff, 6683 NumBytes, Ins, InVals, CS); 6684 } 6685 6686 6687 SDValue PPCTargetLowering::LowerCall_AIX( 6688 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg, 6689 bool isTailCall, bool isPatchPoint, 6690 const SmallVectorImpl<ISD::OutputArg> &Outs, 6691 const SmallVectorImpl<SDValue> &OutVals, 6692 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 6693 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, 6694 ImmutableCallSite CS) const { 6695 6696 assert((CallConv == CallingConv::C || CallConv == CallingConv::Fast) && 6697 "Unimplemented calling convention!"); 6698 if (isVarArg || isPatchPoint) 6699 report_fatal_error("This call type is unimplemented on AIX."); 6700 6701 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 6702 bool isPPC64 = PtrVT == MVT::i64; 6703 unsigned PtrByteSize = isPPC64 ? 8 : 4; 6704 unsigned NumOps = Outs.size(); 6705 6706 6707 // Count how many bytes are to be pushed on the stack, including the linkage 6708 // area, parameter list area. 6709 // On XCOFF, we start with 24/48, which is reserved space for 6710 // [SP][CR][LR][2 x reserved][TOC]. 6711 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 6712 6713 // The prolog code of the callee may store up to 8 GPR argument registers to 6714 // the stack, allowing va_start to index over them in memory if the callee 6715 // is variadic. 6716 // Because we cannot tell if this is needed on the caller side, we have to 6717 // conservatively assume that it is needed. As such, make sure we have at 6718 // least enough stack space for the caller to store the 8 GPRs. 6719 unsigned NumBytes = LinkageSize + 8 * PtrByteSize; 6720 6721 // Adjust the stack pointer for the new arguments... 6722 // These operations are automatically eliminated by the prolog/epilog 6723 // inserter pass. 6724 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); 6725 SDValue CallSeqStart = Chain; 6726 6727 static const MCPhysReg GPR_32[] = { // 32-bit registers. 6728 PPC::R3, PPC::R4, PPC::R5, PPC::R6, 6729 PPC::R7, PPC::R8, PPC::R9, PPC::R10 6730 }; 6731 static const MCPhysReg GPR_64[] = { // 64-bit registers. 6732 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 6733 PPC::X7, PPC::X8, PPC::X9, PPC::X10 6734 }; 6735 6736 const unsigned NumGPRs = isPPC64 ? array_lengthof(GPR_64) 6737 : array_lengthof(GPR_32); 6738 const MCPhysReg *GPR = isPPC64 ? GPR_64 : GPR_32; 6739 unsigned GPR_idx = 0; 6740 6741 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 6742 6743 if (isTailCall) 6744 report_fatal_error("Handling of tail call is unimplemented!"); 6745 int SPDiff = 0; 6746 6747 for (unsigned i = 0; i != NumOps; ++i) { 6748 SDValue Arg = OutVals[i]; 6749 ISD::ArgFlagsTy Flags = Outs[i].Flags; 6750 6751 // Promote integers if needed. 6752 if (Arg.getValueType() == MVT::i1 || 6753 (isPPC64 && Arg.getValueType() == MVT::i32)) { 6754 unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 6755 Arg = DAG.getNode(ExtOp, dl, PtrVT, Arg); 6756 } 6757 6758 // Note: "by value" is code for passing a structure by value, not 6759 // basic types. 6760 if (Flags.isByVal()) 6761 report_fatal_error("Passing structure by value is unimplemented!"); 6762 6763 switch (Arg.getSimpleValueType().SimpleTy) { 6764 default: llvm_unreachable("Unexpected ValueType for argument!"); 6765 case MVT::i1: 6766 case MVT::i32: 6767 case MVT::i64: 6768 if (GPR_idx != NumGPRs) 6769 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Arg)); 6770 else 6771 report_fatal_error("Handling of placing parameters on the stack is " 6772 "unimplemented!"); 6773 break; 6774 case MVT::f32: 6775 case MVT::f64: 6776 case MVT::v4f32: 6777 case MVT::v4i32: 6778 case MVT::v8i16: 6779 case MVT::v16i8: 6780 case MVT::v2f64: 6781 case MVT::v2i64: 6782 case MVT::v1i128: 6783 case MVT::f128: 6784 case MVT::v4f64: 6785 case MVT::v4i1: 6786 report_fatal_error("Handling of this parameter type is unimplemented!"); 6787 } 6788 } 6789 6790 if (!isFunctionGlobalAddress(Callee) && 6791 !isa<ExternalSymbolSDNode>(Callee)) 6792 report_fatal_error("Handling of indirect call is unimplemented!"); 6793 6794 // Build a sequence of copy-to-reg nodes chained together with token chain 6795 // and flag operands which copy the outgoing args into the appropriate regs. 6796 SDValue InFlag; 6797 for (auto Reg : RegsToPass) { 6798 Chain = DAG.getCopyToReg(Chain, dl, Reg.first, Reg.second, InFlag); 6799 InFlag = Chain.getValue(1); 6800 } 6801 6802 return FinishCall(CallConv, dl, isTailCall, isVarArg, isPatchPoint, 6803 /* unused except on PPC64 ELFv1 */ false, DAG, 6804 RegsToPass, InFlag, Chain, CallSeqStart, Callee, SPDiff, 6805 NumBytes, Ins, InVals, CS); 6806 } 6807 6808 bool 6809 PPCTargetLowering::CanLowerReturn(CallingConv::ID CallConv, 6810 MachineFunction &MF, bool isVarArg, 6811 const SmallVectorImpl<ISD::OutputArg> &Outs, 6812 LLVMContext &Context) const { 6813 SmallVector<CCValAssign, 16> RVLocs; 6814 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context); 6815 return CCInfo.CheckReturn( 6816 Outs, (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold) 6817 ? RetCC_PPC_Cold 6818 : RetCC_PPC); 6819 } 6820 6821 SDValue 6822 PPCTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, 6823 bool isVarArg, 6824 const SmallVectorImpl<ISD::OutputArg> &Outs, 6825 const SmallVectorImpl<SDValue> &OutVals, 6826 const SDLoc &dl, SelectionDAG &DAG) const { 6827 SmallVector<CCValAssign, 16> RVLocs; 6828 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, 6829 *DAG.getContext()); 6830 CCInfo.AnalyzeReturn(Outs, 6831 (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold) 6832 ? RetCC_PPC_Cold 6833 : RetCC_PPC); 6834 6835 SDValue Flag; 6836 SmallVector<SDValue, 4> RetOps(1, Chain); 6837 6838 // Copy the result values into the output registers. 6839 for (unsigned i = 0, RealResIdx = 0; i != RVLocs.size(); ++i, ++RealResIdx) { 6840 CCValAssign &VA = RVLocs[i]; 6841 assert(VA.isRegLoc() && "Can only return in registers!"); 6842 6843 SDValue Arg = OutVals[RealResIdx]; 6844 6845 switch (VA.getLocInfo()) { 6846 default: llvm_unreachable("Unknown loc info!"); 6847 case CCValAssign::Full: break; 6848 case CCValAssign::AExt: 6849 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg); 6850 break; 6851 case CCValAssign::ZExt: 6852 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg); 6853 break; 6854 case CCValAssign::SExt: 6855 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg); 6856 break; 6857 } 6858 if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) { 6859 bool isLittleEndian = Subtarget.isLittleEndian(); 6860 // Legalize ret f64 -> ret 2 x i32. 6861 SDValue SVal = 6862 DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg, 6863 DAG.getIntPtrConstant(isLittleEndian ? 0 : 1, dl)); 6864 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), SVal, Flag); 6865 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); 6866 SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg, 6867 DAG.getIntPtrConstant(isLittleEndian ? 1 : 0, dl)); 6868 Flag = Chain.getValue(1); 6869 VA = RVLocs[++i]; // skip ahead to next loc 6870 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), SVal, Flag); 6871 } else 6872 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Arg, Flag); 6873 Flag = Chain.getValue(1); 6874 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); 6875 } 6876 6877 const PPCRegisterInfo *TRI = Subtarget.getRegisterInfo(); 6878 const MCPhysReg *I = 6879 TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction()); 6880 if (I) { 6881 for (; *I; ++I) { 6882 6883 if (PPC::G8RCRegClass.contains(*I)) 6884 RetOps.push_back(DAG.getRegister(*I, MVT::i64)); 6885 else if (PPC::F8RCRegClass.contains(*I)) 6886 RetOps.push_back(DAG.getRegister(*I, MVT::getFloatingPointVT(64))); 6887 else if (PPC::CRRCRegClass.contains(*I)) 6888 RetOps.push_back(DAG.getRegister(*I, MVT::i1)); 6889 else if (PPC::VRRCRegClass.contains(*I)) 6890 RetOps.push_back(DAG.getRegister(*I, MVT::Other)); 6891 else 6892 llvm_unreachable("Unexpected register class in CSRsViaCopy!"); 6893 } 6894 } 6895 6896 RetOps[0] = Chain; // Update chain. 6897 6898 // Add the flag if we have it. 6899 if (Flag.getNode()) 6900 RetOps.push_back(Flag); 6901 6902 return DAG.getNode(PPCISD::RET_FLAG, dl, MVT::Other, RetOps); 6903 } 6904 6905 SDValue 6906 PPCTargetLowering::LowerGET_DYNAMIC_AREA_OFFSET(SDValue Op, 6907 SelectionDAG &DAG) const { 6908 SDLoc dl(Op); 6909 6910 // Get the correct type for integers. 6911 EVT IntVT = Op.getValueType(); 6912 6913 // Get the inputs. 6914 SDValue Chain = Op.getOperand(0); 6915 SDValue FPSIdx = getFramePointerFrameIndex(DAG); 6916 // Build a DYNAREAOFFSET node. 6917 SDValue Ops[2] = {Chain, FPSIdx}; 6918 SDVTList VTs = DAG.getVTList(IntVT); 6919 return DAG.getNode(PPCISD::DYNAREAOFFSET, dl, VTs, Ops); 6920 } 6921 6922 SDValue PPCTargetLowering::LowerSTACKRESTORE(SDValue Op, 6923 SelectionDAG &DAG) const { 6924 // When we pop the dynamic allocation we need to restore the SP link. 6925 SDLoc dl(Op); 6926 6927 // Get the correct type for pointers. 6928 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 6929 6930 // Construct the stack pointer operand. 6931 bool isPPC64 = Subtarget.isPPC64(); 6932 unsigned SP = isPPC64 ? PPC::X1 : PPC::R1; 6933 SDValue StackPtr = DAG.getRegister(SP, PtrVT); 6934 6935 // Get the operands for the STACKRESTORE. 6936 SDValue Chain = Op.getOperand(0); 6937 SDValue SaveSP = Op.getOperand(1); 6938 6939 // Load the old link SP. 6940 SDValue LoadLinkSP = 6941 DAG.getLoad(PtrVT, dl, Chain, StackPtr, MachinePointerInfo()); 6942 6943 // Restore the stack pointer. 6944 Chain = DAG.getCopyToReg(LoadLinkSP.getValue(1), dl, SP, SaveSP); 6945 6946 // Store the old link SP. 6947 return DAG.getStore(Chain, dl, LoadLinkSP, StackPtr, MachinePointerInfo()); 6948 } 6949 6950 SDValue PPCTargetLowering::getReturnAddrFrameIndex(SelectionDAG &DAG) const { 6951 MachineFunction &MF = DAG.getMachineFunction(); 6952 bool isPPC64 = Subtarget.isPPC64(); 6953 EVT PtrVT = getPointerTy(MF.getDataLayout()); 6954 6955 // Get current frame pointer save index. The users of this index will be 6956 // primarily DYNALLOC instructions. 6957 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>(); 6958 int RASI = FI->getReturnAddrSaveIndex(); 6959 6960 // If the frame pointer save index hasn't been defined yet. 6961 if (!RASI) { 6962 // Find out what the fix offset of the frame pointer save area. 6963 int LROffset = Subtarget.getFrameLowering()->getReturnSaveOffset(); 6964 // Allocate the frame index for frame pointer save area. 6965 RASI = MF.getFrameInfo().CreateFixedObject(isPPC64? 8 : 4, LROffset, false); 6966 // Save the result. 6967 FI->setReturnAddrSaveIndex(RASI); 6968 } 6969 return DAG.getFrameIndex(RASI, PtrVT); 6970 } 6971 6972 SDValue 6973 PPCTargetLowering::getFramePointerFrameIndex(SelectionDAG & DAG) const { 6974 MachineFunction &MF = DAG.getMachineFunction(); 6975 bool isPPC64 = Subtarget.isPPC64(); 6976 EVT PtrVT = getPointerTy(MF.getDataLayout()); 6977 6978 // Get current frame pointer save index. The users of this index will be 6979 // primarily DYNALLOC instructions. 6980 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>(); 6981 int FPSI = FI->getFramePointerSaveIndex(); 6982 6983 // If the frame pointer save index hasn't been defined yet. 6984 if (!FPSI) { 6985 // Find out what the fix offset of the frame pointer save area. 6986 int FPOffset = Subtarget.getFrameLowering()->getFramePointerSaveOffset(); 6987 // Allocate the frame index for frame pointer save area. 6988 FPSI = MF.getFrameInfo().CreateFixedObject(isPPC64? 8 : 4, FPOffset, true); 6989 // Save the result. 6990 FI->setFramePointerSaveIndex(FPSI); 6991 } 6992 return DAG.getFrameIndex(FPSI, PtrVT); 6993 } 6994 6995 SDValue PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, 6996 SelectionDAG &DAG) const { 6997 // Get the inputs. 6998 SDValue Chain = Op.getOperand(0); 6999 SDValue Size = Op.getOperand(1); 7000 SDLoc dl(Op); 7001 7002 // Get the correct type for pointers. 7003 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 7004 // Negate the size. 7005 SDValue NegSize = DAG.getNode(ISD::SUB, dl, PtrVT, 7006 DAG.getConstant(0, dl, PtrVT), Size); 7007 // Construct a node for the frame pointer save index. 7008 SDValue FPSIdx = getFramePointerFrameIndex(DAG); 7009 // Build a DYNALLOC node. 7010 SDValue Ops[3] = { Chain, NegSize, FPSIdx }; 7011 SDVTList VTs = DAG.getVTList(PtrVT, MVT::Other); 7012 return DAG.getNode(PPCISD::DYNALLOC, dl, VTs, Ops); 7013 } 7014 7015 SDValue PPCTargetLowering::LowerEH_DWARF_CFA(SDValue Op, 7016 SelectionDAG &DAG) const { 7017 MachineFunction &MF = DAG.getMachineFunction(); 7018 7019 bool isPPC64 = Subtarget.isPPC64(); 7020 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 7021 7022 int FI = MF.getFrameInfo().CreateFixedObject(isPPC64 ? 8 : 4, 0, false); 7023 return DAG.getFrameIndex(FI, PtrVT); 7024 } 7025 7026 SDValue PPCTargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op, 7027 SelectionDAG &DAG) const { 7028 SDLoc DL(Op); 7029 return DAG.getNode(PPCISD::EH_SJLJ_SETJMP, DL, 7030 DAG.getVTList(MVT::i32, MVT::Other), 7031 Op.getOperand(0), Op.getOperand(1)); 7032 } 7033 7034 SDValue PPCTargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op, 7035 SelectionDAG &DAG) const { 7036 SDLoc DL(Op); 7037 return DAG.getNode(PPCISD::EH_SJLJ_LONGJMP, DL, MVT::Other, 7038 Op.getOperand(0), Op.getOperand(1)); 7039 } 7040 7041 SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const { 7042 if (Op.getValueType().isVector()) 7043 return LowerVectorLoad(Op, DAG); 7044 7045 assert(Op.getValueType() == MVT::i1 && 7046 "Custom lowering only for i1 loads"); 7047 7048 // First, load 8 bits into 32 bits, then truncate to 1 bit. 7049 7050 SDLoc dl(Op); 7051 LoadSDNode *LD = cast<LoadSDNode>(Op); 7052 7053 SDValue Chain = LD->getChain(); 7054 SDValue BasePtr = LD->getBasePtr(); 7055 MachineMemOperand *MMO = LD->getMemOperand(); 7056 7057 SDValue NewLD = 7058 DAG.getExtLoad(ISD::EXTLOAD, dl, getPointerTy(DAG.getDataLayout()), Chain, 7059 BasePtr, MVT::i8, MMO); 7060 SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewLD); 7061 7062 SDValue Ops[] = { Result, SDValue(NewLD.getNode(), 1) }; 7063 return DAG.getMergeValues(Ops, dl); 7064 } 7065 7066 SDValue PPCTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const { 7067 if (Op.getOperand(1).getValueType().isVector()) 7068 return LowerVectorStore(Op, DAG); 7069 7070 assert(Op.getOperand(1).getValueType() == MVT::i1 && 7071 "Custom lowering only for i1 stores"); 7072 7073 // First, zero extend to 32 bits, then use a truncating store to 8 bits. 7074 7075 SDLoc dl(Op); 7076 StoreSDNode *ST = cast<StoreSDNode>(Op); 7077 7078 SDValue Chain = ST->getChain(); 7079 SDValue BasePtr = ST->getBasePtr(); 7080 SDValue Value = ST->getValue(); 7081 MachineMemOperand *MMO = ST->getMemOperand(); 7082 7083 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, getPointerTy(DAG.getDataLayout()), 7084 Value); 7085 return DAG.getTruncStore(Chain, dl, Value, BasePtr, MVT::i8, MMO); 7086 } 7087 7088 // FIXME: Remove this once the ANDI glue bug is fixed: 7089 SDValue PPCTargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const { 7090 assert(Op.getValueType() == MVT::i1 && 7091 "Custom lowering only for i1 results"); 7092 7093 SDLoc DL(Op); 7094 return DAG.getNode(PPCISD::ANDIo_1_GT_BIT, DL, MVT::i1, 7095 Op.getOperand(0)); 7096 } 7097 7098 SDValue PPCTargetLowering::LowerTRUNCATEVector(SDValue Op, 7099 SelectionDAG &DAG) const { 7100 7101 // Implements a vector truncate that fits in a vector register as a shuffle. 7102 // We want to legalize vector truncates down to where the source fits in 7103 // a vector register (and target is therefore smaller than vector register 7104 // size). At that point legalization will try to custom lower the sub-legal 7105 // result and get here - where we can contain the truncate as a single target 7106 // operation. 7107 7108 // For example a trunc <2 x i16> to <2 x i8> could be visualized as follows: 7109 // <MSB1|LSB1, MSB2|LSB2> to <LSB1, LSB2> 7110 // 7111 // We will implement it for big-endian ordering as this (where x denotes 7112 // undefined): 7113 // < MSB1|LSB1, MSB2|LSB2, uu, uu, uu, uu, uu, uu> to 7114 // < LSB1, LSB2, u, u, u, u, u, u, u, u, u, u, u, u, u, u> 7115 // 7116 // The same operation in little-endian ordering will be: 7117 // <uu, uu, uu, uu, uu, uu, LSB2|MSB2, LSB1|MSB1> to 7118 // <u, u, u, u, u, u, u, u, u, u, u, u, u, u, LSB2, LSB1> 7119 7120 assert(Op.getValueType().isVector() && "Vector type expected."); 7121 7122 SDLoc DL(Op); 7123 SDValue N1 = Op.getOperand(0); 7124 unsigned SrcSize = N1.getValueType().getSizeInBits(); 7125 assert(SrcSize <= 128 && "Source must fit in an Altivec/VSX vector"); 7126 SDValue WideSrc = SrcSize == 128 ? N1 : widenVec(DAG, N1, DL); 7127 7128 EVT TrgVT = Op.getValueType(); 7129 unsigned TrgNumElts = TrgVT.getVectorNumElements(); 7130 EVT EltVT = TrgVT.getVectorElementType(); 7131 unsigned WideNumElts = 128 / EltVT.getSizeInBits(); 7132 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT, WideNumElts); 7133 7134 // First list the elements we want to keep. 7135 unsigned SizeMult = SrcSize / TrgVT.getSizeInBits(); 7136 SmallVector<int, 16> ShuffV; 7137 if (Subtarget.isLittleEndian()) 7138 for (unsigned i = 0; i < TrgNumElts; ++i) 7139 ShuffV.push_back(i * SizeMult); 7140 else 7141 for (unsigned i = 1; i <= TrgNumElts; ++i) 7142 ShuffV.push_back(i * SizeMult - 1); 7143 7144 // Populate the remaining elements with undefs. 7145 for (unsigned i = TrgNumElts; i < WideNumElts; ++i) 7146 // ShuffV.push_back(i + WideNumElts); 7147 ShuffV.push_back(WideNumElts + 1); 7148 7149 SDValue Conv = DAG.getNode(ISD::BITCAST, DL, WideVT, WideSrc); 7150 return DAG.getVectorShuffle(WideVT, DL, Conv, DAG.getUNDEF(WideVT), ShuffV); 7151 } 7152 7153 /// LowerSELECT_CC - Lower floating point select_cc's into fsel instruction when 7154 /// possible. 7155 SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { 7156 // Not FP? Not a fsel. 7157 if (!Op.getOperand(0).getValueType().isFloatingPoint() || 7158 !Op.getOperand(2).getValueType().isFloatingPoint()) 7159 return Op; 7160 7161 // We might be able to do better than this under some circumstances, but in 7162 // general, fsel-based lowering of select is a finite-math-only optimization. 7163 // For more information, see section F.3 of the 2.06 ISA specification. 7164 if (!DAG.getTarget().Options.NoInfsFPMath || 7165 !DAG.getTarget().Options.NoNaNsFPMath) 7166 return Op; 7167 // TODO: Propagate flags from the select rather than global settings. 7168 SDNodeFlags Flags; 7169 Flags.setNoInfs(true); 7170 Flags.setNoNaNs(true); 7171 7172 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 7173 7174 EVT ResVT = Op.getValueType(); 7175 EVT CmpVT = Op.getOperand(0).getValueType(); 7176 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); 7177 SDValue TV = Op.getOperand(2), FV = Op.getOperand(3); 7178 SDLoc dl(Op); 7179 7180 // If the RHS of the comparison is a 0.0, we don't need to do the 7181 // subtraction at all. 7182 SDValue Sel1; 7183 if (isFloatingPointZero(RHS)) 7184 switch (CC) { 7185 default: break; // SETUO etc aren't handled by fsel. 7186 case ISD::SETNE: 7187 std::swap(TV, FV); 7188 LLVM_FALLTHROUGH; 7189 case ISD::SETEQ: 7190 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits 7191 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); 7192 Sel1 = DAG.getNode(PPCISD::FSEL, dl, ResVT, LHS, TV, FV); 7193 if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits 7194 Sel1 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Sel1); 7195 return DAG.getNode(PPCISD::FSEL, dl, ResVT, 7196 DAG.getNode(ISD::FNEG, dl, MVT::f64, LHS), Sel1, FV); 7197 case ISD::SETULT: 7198 case ISD::SETLT: 7199 std::swap(TV, FV); // fsel is natively setge, swap operands for setlt 7200 LLVM_FALLTHROUGH; 7201 case ISD::SETOGE: 7202 case ISD::SETGE: 7203 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits 7204 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); 7205 return DAG.getNode(PPCISD::FSEL, dl, ResVT, LHS, TV, FV); 7206 case ISD::SETUGT: 7207 case ISD::SETGT: 7208 std::swap(TV, FV); // fsel is natively setge, swap operands for setlt 7209 LLVM_FALLTHROUGH; 7210 case ISD::SETOLE: 7211 case ISD::SETLE: 7212 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits 7213 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); 7214 return DAG.getNode(PPCISD::FSEL, dl, ResVT, 7215 DAG.getNode(ISD::FNEG, dl, MVT::f64, LHS), TV, FV); 7216 } 7217 7218 SDValue Cmp; 7219 switch (CC) { 7220 default: break; // SETUO etc aren't handled by fsel. 7221 case ISD::SETNE: 7222 std::swap(TV, FV); 7223 LLVM_FALLTHROUGH; 7224 case ISD::SETEQ: 7225 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags); 7226 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 7227 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 7228 Sel1 = DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV); 7229 if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits 7230 Sel1 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Sel1); 7231 return DAG.getNode(PPCISD::FSEL, dl, ResVT, 7232 DAG.getNode(ISD::FNEG, dl, MVT::f64, Cmp), Sel1, FV); 7233 case ISD::SETULT: 7234 case ISD::SETLT: 7235 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags); 7236 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 7237 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 7238 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, FV, TV); 7239 case ISD::SETOGE: 7240 case ISD::SETGE: 7241 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags); 7242 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 7243 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 7244 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV); 7245 case ISD::SETUGT: 7246 case ISD::SETGT: 7247 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, RHS, LHS, Flags); 7248 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 7249 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 7250 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, FV, TV); 7251 case ISD::SETOLE: 7252 case ISD::SETLE: 7253 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, RHS, LHS, Flags); 7254 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 7255 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 7256 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV); 7257 } 7258 return Op; 7259 } 7260 7261 void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI, 7262 SelectionDAG &DAG, 7263 const SDLoc &dl) const { 7264 assert(Op.getOperand(0).getValueType().isFloatingPoint()); 7265 SDValue Src = Op.getOperand(0); 7266 if (Src.getValueType() == MVT::f32) 7267 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src); 7268 7269 SDValue Tmp; 7270 switch (Op.getSimpleValueType().SimpleTy) { 7271 default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!"); 7272 case MVT::i32: 7273 Tmp = DAG.getNode( 7274 Op.getOpcode() == ISD::FP_TO_SINT 7275 ? PPCISD::FCTIWZ 7276 : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ), 7277 dl, MVT::f64, Src); 7278 break; 7279 case MVT::i64: 7280 assert((Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()) && 7281 "i64 FP_TO_UINT is supported only with FPCVT"); 7282 Tmp = DAG.getNode(Op.getOpcode()==ISD::FP_TO_SINT ? PPCISD::FCTIDZ : 7283 PPCISD::FCTIDUZ, 7284 dl, MVT::f64, Src); 7285 break; 7286 } 7287 7288 // Convert the FP value to an int value through memory. 7289 bool i32Stack = Op.getValueType() == MVT::i32 && Subtarget.hasSTFIWX() && 7290 (Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()); 7291 SDValue FIPtr = DAG.CreateStackTemporary(i32Stack ? MVT::i32 : MVT::f64); 7292 int FI = cast<FrameIndexSDNode>(FIPtr)->getIndex(); 7293 MachinePointerInfo MPI = 7294 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); 7295 7296 // Emit a store to the stack slot. 7297 SDValue Chain; 7298 if (i32Stack) { 7299 MachineFunction &MF = DAG.getMachineFunction(); 7300 MachineMemOperand *MMO = 7301 MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, 4); 7302 SDValue Ops[] = { DAG.getEntryNode(), Tmp, FIPtr }; 7303 Chain = DAG.getMemIntrinsicNode(PPCISD::STFIWX, dl, 7304 DAG.getVTList(MVT::Other), Ops, MVT::i32, MMO); 7305 } else 7306 Chain = DAG.getStore(DAG.getEntryNode(), dl, Tmp, FIPtr, MPI); 7307 7308 // Result is a load from the stack slot. If loading 4 bytes, make sure to 7309 // add in a bias on big endian. 7310 if (Op.getValueType() == MVT::i32 && !i32Stack) { 7311 FIPtr = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, 7312 DAG.getConstant(4, dl, FIPtr.getValueType())); 7313 MPI = MPI.getWithOffset(Subtarget.isLittleEndian() ? 0 : 4); 7314 } 7315 7316 RLI.Chain = Chain; 7317 RLI.Ptr = FIPtr; 7318 RLI.MPI = MPI; 7319 } 7320 7321 /// Custom lowers floating point to integer conversions to use 7322 /// the direct move instructions available in ISA 2.07 to avoid the 7323 /// need for load/store combinations. 7324 SDValue PPCTargetLowering::LowerFP_TO_INTDirectMove(SDValue Op, 7325 SelectionDAG &DAG, 7326 const SDLoc &dl) const { 7327 assert(Op.getOperand(0).getValueType().isFloatingPoint()); 7328 SDValue Src = Op.getOperand(0); 7329 7330 if (Src.getValueType() == MVT::f32) 7331 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src); 7332 7333 SDValue Tmp; 7334 switch (Op.getSimpleValueType().SimpleTy) { 7335 default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!"); 7336 case MVT::i32: 7337 Tmp = DAG.getNode( 7338 Op.getOpcode() == ISD::FP_TO_SINT 7339 ? PPCISD::FCTIWZ 7340 : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ), 7341 dl, MVT::f64, Src); 7342 Tmp = DAG.getNode(PPCISD::MFVSR, dl, MVT::i32, Tmp); 7343 break; 7344 case MVT::i64: 7345 assert((Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()) && 7346 "i64 FP_TO_UINT is supported only with FPCVT"); 7347 Tmp = DAG.getNode(Op.getOpcode()==ISD::FP_TO_SINT ? PPCISD::FCTIDZ : 7348 PPCISD::FCTIDUZ, 7349 dl, MVT::f64, Src); 7350 Tmp = DAG.getNode(PPCISD::MFVSR, dl, MVT::i64, Tmp); 7351 break; 7352 } 7353 return Tmp; 7354 } 7355 7356 SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG, 7357 const SDLoc &dl) const { 7358 7359 // FP to INT conversions are legal for f128. 7360 if (EnableQuadPrecision && (Op->getOperand(0).getValueType() == MVT::f128)) 7361 return Op; 7362 7363 // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on 7364 // PPC (the libcall is not available). 7365 if (Op.getOperand(0).getValueType() == MVT::ppcf128) { 7366 if (Op.getValueType() == MVT::i32) { 7367 if (Op.getOpcode() == ISD::FP_TO_SINT) { 7368 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, 7369 MVT::f64, Op.getOperand(0), 7370 DAG.getIntPtrConstant(0, dl)); 7371 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, 7372 MVT::f64, Op.getOperand(0), 7373 DAG.getIntPtrConstant(1, dl)); 7374 7375 // Add the two halves of the long double in round-to-zero mode. 7376 SDValue Res = DAG.getNode(PPCISD::FADDRTZ, dl, MVT::f64, Lo, Hi); 7377 7378 // Now use a smaller FP_TO_SINT. 7379 return DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Res); 7380 } 7381 if (Op.getOpcode() == ISD::FP_TO_UINT) { 7382 const uint64_t TwoE31[] = {0x41e0000000000000LL, 0}; 7383 APFloat APF = APFloat(APFloat::PPCDoubleDouble(), APInt(128, TwoE31)); 7384 SDValue Tmp = DAG.getConstantFP(APF, dl, MVT::ppcf128); 7385 // X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X 7386 // FIXME: generated code sucks. 7387 // TODO: Are there fast-math-flags to propagate to this FSUB? 7388 SDValue True = DAG.getNode(ISD::FSUB, dl, MVT::ppcf128, 7389 Op.getOperand(0), Tmp); 7390 True = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, True); 7391 True = DAG.getNode(ISD::ADD, dl, MVT::i32, True, 7392 DAG.getConstant(0x80000000, dl, MVT::i32)); 7393 SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, 7394 Op.getOperand(0)); 7395 return DAG.getSelectCC(dl, Op.getOperand(0), Tmp, True, False, 7396 ISD::SETGE); 7397 } 7398 } 7399 7400 return SDValue(); 7401 } 7402 7403 if (Subtarget.hasDirectMove() && Subtarget.isPPC64()) 7404 return LowerFP_TO_INTDirectMove(Op, DAG, dl); 7405 7406 ReuseLoadInfo RLI; 7407 LowerFP_TO_INTForReuse(Op, RLI, DAG, dl); 7408 7409 return DAG.getLoad(Op.getValueType(), dl, RLI.Chain, RLI.Ptr, RLI.MPI, 7410 RLI.Alignment, RLI.MMOFlags(), RLI.AAInfo, RLI.Ranges); 7411 } 7412 7413 // We're trying to insert a regular store, S, and then a load, L. If the 7414 // incoming value, O, is a load, we might just be able to have our load use the 7415 // address used by O. However, we don't know if anything else will store to 7416 // that address before we can load from it. To prevent this situation, we need 7417 // to insert our load, L, into the chain as a peer of O. To do this, we give L 7418 // the same chain operand as O, we create a token factor from the chain results 7419 // of O and L, and we replace all uses of O's chain result with that token 7420 // factor (see spliceIntoChain below for this last part). 7421 bool PPCTargetLowering::canReuseLoadAddress(SDValue Op, EVT MemVT, 7422 ReuseLoadInfo &RLI, 7423 SelectionDAG &DAG, 7424 ISD::LoadExtType ET) const { 7425 SDLoc dl(Op); 7426 if (ET == ISD::NON_EXTLOAD && 7427 (Op.getOpcode() == ISD::FP_TO_UINT || 7428 Op.getOpcode() == ISD::FP_TO_SINT) && 7429 isOperationLegalOrCustom(Op.getOpcode(), 7430 Op.getOperand(0).getValueType())) { 7431 7432 LowerFP_TO_INTForReuse(Op, RLI, DAG, dl); 7433 return true; 7434 } 7435 7436 LoadSDNode *LD = dyn_cast<LoadSDNode>(Op); 7437 if (!LD || LD->getExtensionType() != ET || LD->isVolatile() || 7438 LD->isNonTemporal()) 7439 return false; 7440 if (LD->getMemoryVT() != MemVT) 7441 return false; 7442 7443 RLI.Ptr = LD->getBasePtr(); 7444 if (LD->isIndexed() && !LD->getOffset().isUndef()) { 7445 assert(LD->getAddressingMode() == ISD::PRE_INC && 7446 "Non-pre-inc AM on PPC?"); 7447 RLI.Ptr = DAG.getNode(ISD::ADD, dl, RLI.Ptr.getValueType(), RLI.Ptr, 7448 LD->getOffset()); 7449 } 7450 7451 RLI.Chain = LD->getChain(); 7452 RLI.MPI = LD->getPointerInfo(); 7453 RLI.IsDereferenceable = LD->isDereferenceable(); 7454 RLI.IsInvariant = LD->isInvariant(); 7455 RLI.Alignment = LD->getAlignment(); 7456 RLI.AAInfo = LD->getAAInfo(); 7457 RLI.Ranges = LD->getRanges(); 7458 7459 RLI.ResChain = SDValue(LD, LD->isIndexed() ? 2 : 1); 7460 return true; 7461 } 7462 7463 // Given the head of the old chain, ResChain, insert a token factor containing 7464 // it and NewResChain, and make users of ResChain now be users of that token 7465 // factor. 7466 // TODO: Remove and use DAG::makeEquivalentMemoryOrdering() instead. 7467 void PPCTargetLowering::spliceIntoChain(SDValue ResChain, 7468 SDValue NewResChain, 7469 SelectionDAG &DAG) const { 7470 if (!ResChain) 7471 return; 7472 7473 SDLoc dl(NewResChain); 7474 7475 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 7476 NewResChain, DAG.getUNDEF(MVT::Other)); 7477 assert(TF.getNode() != NewResChain.getNode() && 7478 "A new TF really is required here"); 7479 7480 DAG.ReplaceAllUsesOfValueWith(ResChain, TF); 7481 DAG.UpdateNodeOperands(TF.getNode(), ResChain, NewResChain); 7482 } 7483 7484 /// Analyze profitability of direct move 7485 /// prefer float load to int load plus direct move 7486 /// when there is no integer use of int load 7487 bool PPCTargetLowering::directMoveIsProfitable(const SDValue &Op) const { 7488 SDNode *Origin = Op.getOperand(0).getNode(); 7489 if (Origin->getOpcode() != ISD::LOAD) 7490 return true; 7491 7492 // If there is no LXSIBZX/LXSIHZX, like Power8, 7493 // prefer direct move if the memory size is 1 or 2 bytes. 7494 MachineMemOperand *MMO = cast<LoadSDNode>(Origin)->getMemOperand(); 7495 if (!Subtarget.hasP9Vector() && MMO->getSize() <= 2) 7496 return true; 7497 7498 for (SDNode::use_iterator UI = Origin->use_begin(), 7499 UE = Origin->use_end(); 7500 UI != UE; ++UI) { 7501 7502 // Only look at the users of the loaded value. 7503 if (UI.getUse().get().getResNo() != 0) 7504 continue; 7505 7506 if (UI->getOpcode() != ISD::SINT_TO_FP && 7507 UI->getOpcode() != ISD::UINT_TO_FP) 7508 return true; 7509 } 7510 7511 return false; 7512 } 7513 7514 /// Custom lowers integer to floating point conversions to use 7515 /// the direct move instructions available in ISA 2.07 to avoid the 7516 /// need for load/store combinations. 7517 SDValue PPCTargetLowering::LowerINT_TO_FPDirectMove(SDValue Op, 7518 SelectionDAG &DAG, 7519 const SDLoc &dl) const { 7520 assert((Op.getValueType() == MVT::f32 || 7521 Op.getValueType() == MVT::f64) && 7522 "Invalid floating point type as target of conversion"); 7523 assert(Subtarget.hasFPCVT() && 7524 "Int to FP conversions with direct moves require FPCVT"); 7525 SDValue FP; 7526 SDValue Src = Op.getOperand(0); 7527 bool SinglePrec = Op.getValueType() == MVT::f32; 7528 bool WordInt = Src.getSimpleValueType().SimpleTy == MVT::i32; 7529 bool Signed = Op.getOpcode() == ISD::SINT_TO_FP; 7530 unsigned ConvOp = Signed ? (SinglePrec ? PPCISD::FCFIDS : PPCISD::FCFID) : 7531 (SinglePrec ? PPCISD::FCFIDUS : PPCISD::FCFIDU); 7532 7533 if (WordInt) { 7534 FP = DAG.getNode(Signed ? PPCISD::MTVSRA : PPCISD::MTVSRZ, 7535 dl, MVT::f64, Src); 7536 FP = DAG.getNode(ConvOp, dl, SinglePrec ? MVT::f32 : MVT::f64, FP); 7537 } 7538 else { 7539 FP = DAG.getNode(PPCISD::MTVSRA, dl, MVT::f64, Src); 7540 FP = DAG.getNode(ConvOp, dl, SinglePrec ? MVT::f32 : MVT::f64, FP); 7541 } 7542 7543 return FP; 7544 } 7545 7546 static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl) { 7547 7548 EVT VecVT = Vec.getValueType(); 7549 assert(VecVT.isVector() && "Expected a vector type."); 7550 assert(VecVT.getSizeInBits() < 128 && "Vector is already full width."); 7551 7552 EVT EltVT = VecVT.getVectorElementType(); 7553 unsigned WideNumElts = 128 / EltVT.getSizeInBits(); 7554 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT, WideNumElts); 7555 7556 unsigned NumConcat = WideNumElts / VecVT.getVectorNumElements(); 7557 SmallVector<SDValue, 16> Ops(NumConcat); 7558 Ops[0] = Vec; 7559 SDValue UndefVec = DAG.getUNDEF(VecVT); 7560 for (unsigned i = 1; i < NumConcat; ++i) 7561 Ops[i] = UndefVec; 7562 7563 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, Ops); 7564 } 7565 7566 SDValue PPCTargetLowering::LowerINT_TO_FPVector(SDValue Op, SelectionDAG &DAG, 7567 const SDLoc &dl) const { 7568 7569 unsigned Opc = Op.getOpcode(); 7570 assert((Opc == ISD::UINT_TO_FP || Opc == ISD::SINT_TO_FP) && 7571 "Unexpected conversion type"); 7572 assert((Op.getValueType() == MVT::v2f64 || Op.getValueType() == MVT::v4f32) && 7573 "Supports conversions to v2f64/v4f32 only."); 7574 7575 bool SignedConv = Opc == ISD::SINT_TO_FP; 7576 bool FourEltRes = Op.getValueType() == MVT::v4f32; 7577 7578 SDValue Wide = widenVec(DAG, Op.getOperand(0), dl); 7579 EVT WideVT = Wide.getValueType(); 7580 unsigned WideNumElts = WideVT.getVectorNumElements(); 7581 MVT IntermediateVT = FourEltRes ? MVT::v4i32 : MVT::v2i64; 7582 7583 SmallVector<int, 16> ShuffV; 7584 for (unsigned i = 0; i < WideNumElts; ++i) 7585 ShuffV.push_back(i + WideNumElts); 7586 7587 int Stride = FourEltRes ? WideNumElts / 4 : WideNumElts / 2; 7588 int SaveElts = FourEltRes ? 4 : 2; 7589 if (Subtarget.isLittleEndian()) 7590 for (int i = 0; i < SaveElts; i++) 7591 ShuffV[i * Stride] = i; 7592 else 7593 for (int i = 1; i <= SaveElts; i++) 7594 ShuffV[i * Stride - 1] = i - 1; 7595 7596 SDValue ShuffleSrc2 = 7597 SignedConv ? DAG.getUNDEF(WideVT) : DAG.getConstant(0, dl, WideVT); 7598 SDValue Arrange = DAG.getVectorShuffle(WideVT, dl, Wide, ShuffleSrc2, ShuffV); 7599 unsigned ExtendOp = 7600 SignedConv ? (unsigned)PPCISD::SExtVElems : (unsigned)ISD::BITCAST; 7601 7602 SDValue Extend; 7603 if (!Subtarget.hasP9Altivec() && SignedConv) { 7604 Arrange = DAG.getBitcast(IntermediateVT, Arrange); 7605 Extend = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, IntermediateVT, Arrange, 7606 DAG.getValueType(Op.getOperand(0).getValueType())); 7607 } else 7608 Extend = DAG.getNode(ExtendOp, dl, IntermediateVT, Arrange); 7609 7610 return DAG.getNode(Opc, dl, Op.getValueType(), Extend); 7611 } 7612 7613 SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op, 7614 SelectionDAG &DAG) const { 7615 SDLoc dl(Op); 7616 7617 EVT InVT = Op.getOperand(0).getValueType(); 7618 EVT OutVT = Op.getValueType(); 7619 if (OutVT.isVector() && OutVT.isFloatingPoint() && 7620 isOperationCustom(Op.getOpcode(), InVT)) 7621 return LowerINT_TO_FPVector(Op, DAG, dl); 7622 7623 // Conversions to f128 are legal. 7624 if (EnableQuadPrecision && (Op.getValueType() == MVT::f128)) 7625 return Op; 7626 7627 if (Subtarget.hasQPX() && Op.getOperand(0).getValueType() == MVT::v4i1) { 7628 if (Op.getValueType() != MVT::v4f32 && Op.getValueType() != MVT::v4f64) 7629 return SDValue(); 7630 7631 SDValue Value = Op.getOperand(0); 7632 // The values are now known to be -1 (false) or 1 (true). To convert this 7633 // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5). 7634 // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5 7635 Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value); 7636 7637 SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64); 7638 7639 Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs); 7640 7641 if (Op.getValueType() != MVT::v4f64) 7642 Value = DAG.getNode(ISD::FP_ROUND, dl, 7643 Op.getValueType(), Value, 7644 DAG.getIntPtrConstant(1, dl)); 7645 return Value; 7646 } 7647 7648 // Don't handle ppc_fp128 here; let it be lowered to a libcall. 7649 if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64) 7650 return SDValue(); 7651 7652 if (Op.getOperand(0).getValueType() == MVT::i1) 7653 return DAG.getNode(ISD::SELECT, dl, Op.getValueType(), Op.getOperand(0), 7654 DAG.getConstantFP(1.0, dl, Op.getValueType()), 7655 DAG.getConstantFP(0.0, dl, Op.getValueType())); 7656 7657 // If we have direct moves, we can do all the conversion, skip the store/load 7658 // however, without FPCVT we can't do most conversions. 7659 if (Subtarget.hasDirectMove() && directMoveIsProfitable(Op) && 7660 Subtarget.isPPC64() && Subtarget.hasFPCVT()) 7661 return LowerINT_TO_FPDirectMove(Op, DAG, dl); 7662 7663 assert((Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) && 7664 "UINT_TO_FP is supported only with FPCVT"); 7665 7666 // If we have FCFIDS, then use it when converting to single-precision. 7667 // Otherwise, convert to double-precision and then round. 7668 unsigned FCFOp = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) 7669 ? (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDUS 7670 : PPCISD::FCFIDS) 7671 : (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDU 7672 : PPCISD::FCFID); 7673 MVT FCFTy = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) 7674 ? MVT::f32 7675 : MVT::f64; 7676 7677 if (Op.getOperand(0).getValueType() == MVT::i64) { 7678 SDValue SINT = Op.getOperand(0); 7679 // When converting to single-precision, we actually need to convert 7680 // to double-precision first and then round to single-precision. 7681 // To avoid double-rounding effects during that operation, we have 7682 // to prepare the input operand. Bits that might be truncated when 7683 // converting to double-precision are replaced by a bit that won't 7684 // be lost at this stage, but is below the single-precision rounding 7685 // position. 7686 // 7687 // However, if -enable-unsafe-fp-math is in effect, accept double 7688 // rounding to avoid the extra overhead. 7689 if (Op.getValueType() == MVT::f32 && 7690 !Subtarget.hasFPCVT() && 7691 !DAG.getTarget().Options.UnsafeFPMath) { 7692 7693 // Twiddle input to make sure the low 11 bits are zero. (If this 7694 // is the case, we are guaranteed the value will fit into the 53 bit 7695 // mantissa of an IEEE double-precision value without rounding.) 7696 // If any of those low 11 bits were not zero originally, make sure 7697 // bit 12 (value 2048) is set instead, so that the final rounding 7698 // to single-precision gets the correct result. 7699 SDValue Round = DAG.getNode(ISD::AND, dl, MVT::i64, 7700 SINT, DAG.getConstant(2047, dl, MVT::i64)); 7701 Round = DAG.getNode(ISD::ADD, dl, MVT::i64, 7702 Round, DAG.getConstant(2047, dl, MVT::i64)); 7703 Round = DAG.getNode(ISD::OR, dl, MVT::i64, Round, SINT); 7704 Round = DAG.getNode(ISD::AND, dl, MVT::i64, 7705 Round, DAG.getConstant(-2048, dl, MVT::i64)); 7706 7707 // However, we cannot use that value unconditionally: if the magnitude 7708 // of the input value is small, the bit-twiddling we did above might 7709 // end up visibly changing the output. Fortunately, in that case, we 7710 // don't need to twiddle bits since the original input will convert 7711 // exactly to double-precision floating-point already. Therefore, 7712 // construct a conditional to use the original value if the top 11 7713 // bits are all sign-bit copies, and use the rounded value computed 7714 // above otherwise. 7715 SDValue Cond = DAG.getNode(ISD::SRA, dl, MVT::i64, 7716 SINT, DAG.getConstant(53, dl, MVT::i32)); 7717 Cond = DAG.getNode(ISD::ADD, dl, MVT::i64, 7718 Cond, DAG.getConstant(1, dl, MVT::i64)); 7719 Cond = DAG.getSetCC(dl, MVT::i32, 7720 Cond, DAG.getConstant(1, dl, MVT::i64), ISD::SETUGT); 7721 7722 SINT = DAG.getNode(ISD::SELECT, dl, MVT::i64, Cond, Round, SINT); 7723 } 7724 7725 ReuseLoadInfo RLI; 7726 SDValue Bits; 7727 7728 MachineFunction &MF = DAG.getMachineFunction(); 7729 if (canReuseLoadAddress(SINT, MVT::i64, RLI, DAG)) { 7730 Bits = DAG.getLoad(MVT::f64, dl, RLI.Chain, RLI.Ptr, RLI.MPI, 7731 RLI.Alignment, RLI.MMOFlags(), RLI.AAInfo, RLI.Ranges); 7732 spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG); 7733 } else if (Subtarget.hasLFIWAX() && 7734 canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::SEXTLOAD)) { 7735 MachineMemOperand *MMO = 7736 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, 7737 RLI.Alignment, RLI.AAInfo, RLI.Ranges); 7738 SDValue Ops[] = { RLI.Chain, RLI.Ptr }; 7739 Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWAX, dl, 7740 DAG.getVTList(MVT::f64, MVT::Other), 7741 Ops, MVT::i32, MMO); 7742 spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG); 7743 } else if (Subtarget.hasFPCVT() && 7744 canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::ZEXTLOAD)) { 7745 MachineMemOperand *MMO = 7746 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, 7747 RLI.Alignment, RLI.AAInfo, RLI.Ranges); 7748 SDValue Ops[] = { RLI.Chain, RLI.Ptr }; 7749 Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWZX, dl, 7750 DAG.getVTList(MVT::f64, MVT::Other), 7751 Ops, MVT::i32, MMO); 7752 spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG); 7753 } else if (((Subtarget.hasLFIWAX() && 7754 SINT.getOpcode() == ISD::SIGN_EXTEND) || 7755 (Subtarget.hasFPCVT() && 7756 SINT.getOpcode() == ISD::ZERO_EXTEND)) && 7757 SINT.getOperand(0).getValueType() == MVT::i32) { 7758 MachineFrameInfo &MFI = MF.getFrameInfo(); 7759 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 7760 7761 int FrameIdx = MFI.CreateStackObject(4, 4, false); 7762 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 7763 7764 SDValue Store = 7765 DAG.getStore(DAG.getEntryNode(), dl, SINT.getOperand(0), FIdx, 7766 MachinePointerInfo::getFixedStack( 7767 DAG.getMachineFunction(), FrameIdx)); 7768 7769 assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 && 7770 "Expected an i32 store"); 7771 7772 RLI.Ptr = FIdx; 7773 RLI.Chain = Store; 7774 RLI.MPI = 7775 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 7776 RLI.Alignment = 4; 7777 7778 MachineMemOperand *MMO = 7779 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, 7780 RLI.Alignment, RLI.AAInfo, RLI.Ranges); 7781 SDValue Ops[] = { RLI.Chain, RLI.Ptr }; 7782 Bits = DAG.getMemIntrinsicNode(SINT.getOpcode() == ISD::ZERO_EXTEND ? 7783 PPCISD::LFIWZX : PPCISD::LFIWAX, 7784 dl, DAG.getVTList(MVT::f64, MVT::Other), 7785 Ops, MVT::i32, MMO); 7786 } else 7787 Bits = DAG.getNode(ISD::BITCAST, dl, MVT::f64, SINT); 7788 7789 SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Bits); 7790 7791 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) 7792 FP = DAG.getNode(ISD::FP_ROUND, dl, 7793 MVT::f32, FP, DAG.getIntPtrConstant(0, dl)); 7794 return FP; 7795 } 7796 7797 assert(Op.getOperand(0).getValueType() == MVT::i32 && 7798 "Unhandled INT_TO_FP type in custom expander!"); 7799 // Since we only generate this in 64-bit mode, we can take advantage of 7800 // 64-bit registers. In particular, sign extend the input value into the 7801 // 64-bit register with extsw, store the WHOLE 64-bit value into the stack 7802 // then lfd it and fcfid it. 7803 MachineFunction &MF = DAG.getMachineFunction(); 7804 MachineFrameInfo &MFI = MF.getFrameInfo(); 7805 EVT PtrVT = getPointerTy(MF.getDataLayout()); 7806 7807 SDValue Ld; 7808 if (Subtarget.hasLFIWAX() || Subtarget.hasFPCVT()) { 7809 ReuseLoadInfo RLI; 7810 bool ReusingLoad; 7811 if (!(ReusingLoad = canReuseLoadAddress(Op.getOperand(0), MVT::i32, RLI, 7812 DAG))) { 7813 int FrameIdx = MFI.CreateStackObject(4, 4, false); 7814 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 7815 7816 SDValue Store = 7817 DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), FIdx, 7818 MachinePointerInfo::getFixedStack( 7819 DAG.getMachineFunction(), FrameIdx)); 7820 7821 assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 && 7822 "Expected an i32 store"); 7823 7824 RLI.Ptr = FIdx; 7825 RLI.Chain = Store; 7826 RLI.MPI = 7827 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 7828 RLI.Alignment = 4; 7829 } 7830 7831 MachineMemOperand *MMO = 7832 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, 7833 RLI.Alignment, RLI.AAInfo, RLI.Ranges); 7834 SDValue Ops[] = { RLI.Chain, RLI.Ptr }; 7835 Ld = DAG.getMemIntrinsicNode(Op.getOpcode() == ISD::UINT_TO_FP ? 7836 PPCISD::LFIWZX : PPCISD::LFIWAX, 7837 dl, DAG.getVTList(MVT::f64, MVT::Other), 7838 Ops, MVT::i32, MMO); 7839 if (ReusingLoad) 7840 spliceIntoChain(RLI.ResChain, Ld.getValue(1), DAG); 7841 } else { 7842 assert(Subtarget.isPPC64() && 7843 "i32->FP without LFIWAX supported only on PPC64"); 7844 7845 int FrameIdx = MFI.CreateStackObject(8, 8, false); 7846 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 7847 7848 SDValue Ext64 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i64, 7849 Op.getOperand(0)); 7850 7851 // STD the extended value into the stack slot. 7852 SDValue Store = DAG.getStore( 7853 DAG.getEntryNode(), dl, Ext64, FIdx, 7854 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx)); 7855 7856 // Load the value as a double. 7857 Ld = DAG.getLoad( 7858 MVT::f64, dl, Store, FIdx, 7859 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx)); 7860 } 7861 7862 // FCFID it and return it. 7863 SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Ld); 7864 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) 7865 FP = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, FP, 7866 DAG.getIntPtrConstant(0, dl)); 7867 return FP; 7868 } 7869 7870 SDValue PPCTargetLowering::LowerFLT_ROUNDS_(SDValue Op, 7871 SelectionDAG &DAG) const { 7872 SDLoc dl(Op); 7873 /* 7874 The rounding mode is in bits 30:31 of FPSR, and has the following 7875 settings: 7876 00 Round to nearest 7877 01 Round to 0 7878 10 Round to +inf 7879 11 Round to -inf 7880 7881 FLT_ROUNDS, on the other hand, expects the following: 7882 -1 Undefined 7883 0 Round to 0 7884 1 Round to nearest 7885 2 Round to +inf 7886 3 Round to -inf 7887 7888 To perform the conversion, we do: 7889 ((FPSCR & 0x3) ^ ((~FPSCR & 0x3) >> 1)) 7890 */ 7891 7892 MachineFunction &MF = DAG.getMachineFunction(); 7893 EVT VT = Op.getValueType(); 7894 EVT PtrVT = getPointerTy(MF.getDataLayout()); 7895 7896 // Save FP Control Word to register 7897 EVT NodeTys[] = { 7898 MVT::f64, // return register 7899 MVT::Glue // unused in this context 7900 }; 7901 SDValue Chain = DAG.getNode(PPCISD::MFFS, dl, NodeTys, None); 7902 7903 // Save FP register to stack slot 7904 int SSFI = MF.getFrameInfo().CreateStackObject(8, 8, false); 7905 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT); 7906 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Chain, StackSlot, 7907 MachinePointerInfo()); 7908 7909 // Load FP Control Word from low 32 bits of stack slot. 7910 SDValue Four = DAG.getConstant(4, dl, PtrVT); 7911 SDValue Addr = DAG.getNode(ISD::ADD, dl, PtrVT, StackSlot, Four); 7912 SDValue CWD = DAG.getLoad(MVT::i32, dl, Store, Addr, MachinePointerInfo()); 7913 7914 // Transform as necessary 7915 SDValue CWD1 = 7916 DAG.getNode(ISD::AND, dl, MVT::i32, 7917 CWD, DAG.getConstant(3, dl, MVT::i32)); 7918 SDValue CWD2 = 7919 DAG.getNode(ISD::SRL, dl, MVT::i32, 7920 DAG.getNode(ISD::AND, dl, MVT::i32, 7921 DAG.getNode(ISD::XOR, dl, MVT::i32, 7922 CWD, DAG.getConstant(3, dl, MVT::i32)), 7923 DAG.getConstant(3, dl, MVT::i32)), 7924 DAG.getConstant(1, dl, MVT::i32)); 7925 7926 SDValue RetVal = 7927 DAG.getNode(ISD::XOR, dl, MVT::i32, CWD1, CWD2); 7928 7929 return DAG.getNode((VT.getSizeInBits() < 16 ? 7930 ISD::TRUNCATE : ISD::ZERO_EXTEND), dl, VT, RetVal); 7931 } 7932 7933 SDValue PPCTargetLowering::LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG) const { 7934 EVT VT = Op.getValueType(); 7935 unsigned BitWidth = VT.getSizeInBits(); 7936 SDLoc dl(Op); 7937 assert(Op.getNumOperands() == 3 && 7938 VT == Op.getOperand(1).getValueType() && 7939 "Unexpected SHL!"); 7940 7941 // Expand into a bunch of logical ops. Note that these ops 7942 // depend on the PPC behavior for oversized shift amounts. 7943 SDValue Lo = Op.getOperand(0); 7944 SDValue Hi = Op.getOperand(1); 7945 SDValue Amt = Op.getOperand(2); 7946 EVT AmtVT = Amt.getValueType(); 7947 7948 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT, 7949 DAG.getConstant(BitWidth, dl, AmtVT), Amt); 7950 SDValue Tmp2 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Amt); 7951 SDValue Tmp3 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Tmp1); 7952 SDValue Tmp4 = DAG.getNode(ISD::OR , dl, VT, Tmp2, Tmp3); 7953 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt, 7954 DAG.getConstant(-BitWidth, dl, AmtVT)); 7955 SDValue Tmp6 = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Tmp5); 7956 SDValue OutHi = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6); 7957 SDValue OutLo = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Amt); 7958 SDValue OutOps[] = { OutLo, OutHi }; 7959 return DAG.getMergeValues(OutOps, dl); 7960 } 7961 7962 SDValue PPCTargetLowering::LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const { 7963 EVT VT = Op.getValueType(); 7964 SDLoc dl(Op); 7965 unsigned BitWidth = VT.getSizeInBits(); 7966 assert(Op.getNumOperands() == 3 && 7967 VT == Op.getOperand(1).getValueType() && 7968 "Unexpected SRL!"); 7969 7970 // Expand into a bunch of logical ops. Note that these ops 7971 // depend on the PPC behavior for oversized shift amounts. 7972 SDValue Lo = Op.getOperand(0); 7973 SDValue Hi = Op.getOperand(1); 7974 SDValue Amt = Op.getOperand(2); 7975 EVT AmtVT = Amt.getValueType(); 7976 7977 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT, 7978 DAG.getConstant(BitWidth, dl, AmtVT), Amt); 7979 SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt); 7980 SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1); 7981 SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); 7982 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt, 7983 DAG.getConstant(-BitWidth, dl, AmtVT)); 7984 SDValue Tmp6 = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Tmp5); 7985 SDValue OutLo = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6); 7986 SDValue OutHi = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Amt); 7987 SDValue OutOps[] = { OutLo, OutHi }; 7988 return DAG.getMergeValues(OutOps, dl); 7989 } 7990 7991 SDValue PPCTargetLowering::LowerSRA_PARTS(SDValue Op, SelectionDAG &DAG) const { 7992 SDLoc dl(Op); 7993 EVT VT = Op.getValueType(); 7994 unsigned BitWidth = VT.getSizeInBits(); 7995 assert(Op.getNumOperands() == 3 && 7996 VT == Op.getOperand(1).getValueType() && 7997 "Unexpected SRA!"); 7998 7999 // Expand into a bunch of logical ops, followed by a select_cc. 8000 SDValue Lo = Op.getOperand(0); 8001 SDValue Hi = Op.getOperand(1); 8002 SDValue Amt = Op.getOperand(2); 8003 EVT AmtVT = Amt.getValueType(); 8004 8005 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT, 8006 DAG.getConstant(BitWidth, dl, AmtVT), Amt); 8007 SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt); 8008 SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1); 8009 SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); 8010 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt, 8011 DAG.getConstant(-BitWidth, dl, AmtVT)); 8012 SDValue Tmp6 = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Tmp5); 8013 SDValue OutHi = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Amt); 8014 SDValue OutLo = DAG.getSelectCC(dl, Tmp5, DAG.getConstant(0, dl, AmtVT), 8015 Tmp4, Tmp6, ISD::SETLE); 8016 SDValue OutOps[] = { OutLo, OutHi }; 8017 return DAG.getMergeValues(OutOps, dl); 8018 } 8019 8020 //===----------------------------------------------------------------------===// 8021 // Vector related lowering. 8022 // 8023 8024 /// BuildSplatI - Build a canonical splati of Val with an element size of 8025 /// SplatSize. Cast the result to VT. 8026 static SDValue BuildSplatI(int Val, unsigned SplatSize, EVT VT, 8027 SelectionDAG &DAG, const SDLoc &dl) { 8028 assert(Val >= -16 && Val <= 15 && "vsplti is out of range!"); 8029 8030 static const MVT VTys[] = { // canonical VT to use for each size. 8031 MVT::v16i8, MVT::v8i16, MVT::Other, MVT::v4i32 8032 }; 8033 8034 EVT ReqVT = VT != MVT::Other ? VT : VTys[SplatSize-1]; 8035 8036 // Force vspltis[hw] -1 to vspltisb -1 to canonicalize. 8037 if (Val == -1) 8038 SplatSize = 1; 8039 8040 EVT CanonicalVT = VTys[SplatSize-1]; 8041 8042 // Build a canonical splat for this value. 8043 return DAG.getBitcast(ReqVT, DAG.getConstant(Val, dl, CanonicalVT)); 8044 } 8045 8046 /// BuildIntrinsicOp - Return a unary operator intrinsic node with the 8047 /// specified intrinsic ID. 8048 static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op, SelectionDAG &DAG, 8049 const SDLoc &dl, EVT DestVT = MVT::Other) { 8050 if (DestVT == MVT::Other) DestVT = Op.getValueType(); 8051 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT, 8052 DAG.getConstant(IID, dl, MVT::i32), Op); 8053 } 8054 8055 /// BuildIntrinsicOp - Return a binary operator intrinsic node with the 8056 /// specified intrinsic ID. 8057 static SDValue BuildIntrinsicOp(unsigned IID, SDValue LHS, SDValue RHS, 8058 SelectionDAG &DAG, const SDLoc &dl, 8059 EVT DestVT = MVT::Other) { 8060 if (DestVT == MVT::Other) DestVT = LHS.getValueType(); 8061 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT, 8062 DAG.getConstant(IID, dl, MVT::i32), LHS, RHS); 8063 } 8064 8065 /// BuildIntrinsicOp - Return a ternary operator intrinsic node with the 8066 /// specified intrinsic ID. 8067 static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op0, SDValue Op1, 8068 SDValue Op2, SelectionDAG &DAG, const SDLoc &dl, 8069 EVT DestVT = MVT::Other) { 8070 if (DestVT == MVT::Other) DestVT = Op0.getValueType(); 8071 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT, 8072 DAG.getConstant(IID, dl, MVT::i32), Op0, Op1, Op2); 8073 } 8074 8075 /// BuildVSLDOI - Return a VECTOR_SHUFFLE that is a vsldoi of the specified 8076 /// amount. The result has the specified value type. 8077 static SDValue BuildVSLDOI(SDValue LHS, SDValue RHS, unsigned Amt, EVT VT, 8078 SelectionDAG &DAG, const SDLoc &dl) { 8079 // Force LHS/RHS to be the right type. 8080 LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, LHS); 8081 RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, RHS); 8082 8083 int Ops[16]; 8084 for (unsigned i = 0; i != 16; ++i) 8085 Ops[i] = i + Amt; 8086 SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, LHS, RHS, Ops); 8087 return DAG.getNode(ISD::BITCAST, dl, VT, T); 8088 } 8089 8090 /// Do we have an efficient pattern in a .td file for this node? 8091 /// 8092 /// \param V - pointer to the BuildVectorSDNode being matched 8093 /// \param HasDirectMove - does this subtarget have VSR <-> GPR direct moves? 8094 /// 8095 /// There are some patterns where it is beneficial to keep a BUILD_VECTOR 8096 /// node as a BUILD_VECTOR node rather than expanding it. The patterns where 8097 /// the opposite is true (expansion is beneficial) are: 8098 /// - The node builds a vector out of integers that are not 32 or 64-bits 8099 /// - The node builds a vector out of constants 8100 /// - The node is a "load-and-splat" 8101 /// In all other cases, we will choose to keep the BUILD_VECTOR. 8102 static bool haveEfficientBuildVectorPattern(BuildVectorSDNode *V, 8103 bool HasDirectMove, 8104 bool HasP8Vector) { 8105 EVT VecVT = V->getValueType(0); 8106 bool RightType = VecVT == MVT::v2f64 || 8107 (HasP8Vector && VecVT == MVT::v4f32) || 8108 (HasDirectMove && (VecVT == MVT::v2i64 || VecVT == MVT::v4i32)); 8109 if (!RightType) 8110 return false; 8111 8112 bool IsSplat = true; 8113 bool IsLoad = false; 8114 SDValue Op0 = V->getOperand(0); 8115 8116 // This function is called in a block that confirms the node is not a constant 8117 // splat. So a constant BUILD_VECTOR here means the vector is built out of 8118 // different constants. 8119 if (V->isConstant()) 8120 return false; 8121 for (int i = 0, e = V->getNumOperands(); i < e; ++i) { 8122 if (V->getOperand(i).isUndef()) 8123 return false; 8124 // We want to expand nodes that represent load-and-splat even if the 8125 // loaded value is a floating point truncation or conversion to int. 8126 if (V->getOperand(i).getOpcode() == ISD::LOAD || 8127 (V->getOperand(i).getOpcode() == ISD::FP_ROUND && 8128 V->getOperand(i).getOperand(0).getOpcode() == ISD::LOAD) || 8129 (V->getOperand(i).getOpcode() == ISD::FP_TO_SINT && 8130 V->getOperand(i).getOperand(0).getOpcode() == ISD::LOAD) || 8131 (V->getOperand(i).getOpcode() == ISD::FP_TO_UINT && 8132 V->getOperand(i).getOperand(0).getOpcode() == ISD::LOAD)) 8133 IsLoad = true; 8134 // If the operands are different or the input is not a load and has more 8135 // uses than just this BV node, then it isn't a splat. 8136 if (V->getOperand(i) != Op0 || 8137 (!IsLoad && !V->isOnlyUserOf(V->getOperand(i).getNode()))) 8138 IsSplat = false; 8139 } 8140 return !(IsSplat && IsLoad); 8141 } 8142 8143 // Lower BITCAST(f128, (build_pair i64, i64)) to BUILD_FP128. 8144 SDValue PPCTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const { 8145 8146 SDLoc dl(Op); 8147 SDValue Op0 = Op->getOperand(0); 8148 8149 if (!EnableQuadPrecision || 8150 (Op.getValueType() != MVT::f128 ) || 8151 (Op0.getOpcode() != ISD::BUILD_PAIR) || 8152 (Op0.getOperand(0).getValueType() != MVT::i64) || 8153 (Op0.getOperand(1).getValueType() != MVT::i64)) 8154 return SDValue(); 8155 8156 return DAG.getNode(PPCISD::BUILD_FP128, dl, MVT::f128, Op0.getOperand(0), 8157 Op0.getOperand(1)); 8158 } 8159 8160 // If this is a case we can't handle, return null and let the default 8161 // expansion code take care of it. If we CAN select this case, and if it 8162 // selects to a single instruction, return Op. Otherwise, if we can codegen 8163 // this case more efficiently than a constant pool load, lower it to the 8164 // sequence of ops that should be used. 8165 SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op, 8166 SelectionDAG &DAG) const { 8167 SDLoc dl(Op); 8168 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode()); 8169 assert(BVN && "Expected a BuildVectorSDNode in LowerBUILD_VECTOR"); 8170 8171 if (Subtarget.hasQPX() && Op.getValueType() == MVT::v4i1) { 8172 // We first build an i32 vector, load it into a QPX register, 8173 // then convert it to a floating-point vector and compare it 8174 // to a zero vector to get the boolean result. 8175 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); 8176 int FrameIdx = MFI.CreateStackObject(16, 16, false); 8177 MachinePointerInfo PtrInfo = 8178 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 8179 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 8180 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 8181 8182 assert(BVN->getNumOperands() == 4 && 8183 "BUILD_VECTOR for v4i1 does not have 4 operands"); 8184 8185 bool IsConst = true; 8186 for (unsigned i = 0; i < 4; ++i) { 8187 if (BVN->getOperand(i).isUndef()) continue; 8188 if (!isa<ConstantSDNode>(BVN->getOperand(i))) { 8189 IsConst = false; 8190 break; 8191 } 8192 } 8193 8194 if (IsConst) { 8195 Constant *One = 8196 ConstantFP::get(Type::getFloatTy(*DAG.getContext()), 1.0); 8197 Constant *NegOne = 8198 ConstantFP::get(Type::getFloatTy(*DAG.getContext()), -1.0); 8199 8200 Constant *CV[4]; 8201 for (unsigned i = 0; i < 4; ++i) { 8202 if (BVN->getOperand(i).isUndef()) 8203 CV[i] = UndefValue::get(Type::getFloatTy(*DAG.getContext())); 8204 else if (isNullConstant(BVN->getOperand(i))) 8205 CV[i] = NegOne; 8206 else 8207 CV[i] = One; 8208 } 8209 8210 Constant *CP = ConstantVector::get(CV); 8211 SDValue CPIdx = DAG.getConstantPool(CP, getPointerTy(DAG.getDataLayout()), 8212 16 /* alignment */); 8213 8214 SDValue Ops[] = {DAG.getEntryNode(), CPIdx}; 8215 SDVTList VTs = DAG.getVTList({MVT::v4i1, /*chain*/ MVT::Other}); 8216 return DAG.getMemIntrinsicNode( 8217 PPCISD::QVLFSb, dl, VTs, Ops, MVT::v4f32, 8218 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 8219 } 8220 8221 SmallVector<SDValue, 4> Stores; 8222 for (unsigned i = 0; i < 4; ++i) { 8223 if (BVN->getOperand(i).isUndef()) continue; 8224 8225 unsigned Offset = 4*i; 8226 SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType()); 8227 Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx); 8228 8229 unsigned StoreSize = BVN->getOperand(i).getValueType().getStoreSize(); 8230 if (StoreSize > 4) { 8231 Stores.push_back( 8232 DAG.getTruncStore(DAG.getEntryNode(), dl, BVN->getOperand(i), Idx, 8233 PtrInfo.getWithOffset(Offset), MVT::i32)); 8234 } else { 8235 SDValue StoreValue = BVN->getOperand(i); 8236 if (StoreSize < 4) 8237 StoreValue = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, StoreValue); 8238 8239 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, StoreValue, Idx, 8240 PtrInfo.getWithOffset(Offset))); 8241 } 8242 } 8243 8244 SDValue StoreChain; 8245 if (!Stores.empty()) 8246 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); 8247 else 8248 StoreChain = DAG.getEntryNode(); 8249 8250 // Now load from v4i32 into the QPX register; this will extend it to 8251 // v4i64 but not yet convert it to a floating point. Nevertheless, this 8252 // is typed as v4f64 because the QPX register integer states are not 8253 // explicitly represented. 8254 8255 SDValue Ops[] = {StoreChain, 8256 DAG.getConstant(Intrinsic::ppc_qpx_qvlfiwz, dl, MVT::i32), 8257 FIdx}; 8258 SDVTList VTs = DAG.getVTList({MVT::v4f64, /*chain*/ MVT::Other}); 8259 8260 SDValue LoadedVect = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, 8261 dl, VTs, Ops, MVT::v4i32, PtrInfo); 8262 LoadedVect = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64, 8263 DAG.getConstant(Intrinsic::ppc_qpx_qvfcfidu, dl, MVT::i32), 8264 LoadedVect); 8265 8266 SDValue FPZeros = DAG.getConstantFP(0.0, dl, MVT::v4f64); 8267 8268 return DAG.getSetCC(dl, MVT::v4i1, LoadedVect, FPZeros, ISD::SETEQ); 8269 } 8270 8271 // All other QPX vectors are handled by generic code. 8272 if (Subtarget.hasQPX()) 8273 return SDValue(); 8274 8275 // Check if this is a splat of a constant value. 8276 APInt APSplatBits, APSplatUndef; 8277 unsigned SplatBitSize; 8278 bool HasAnyUndefs; 8279 if (! BVN->isConstantSplat(APSplatBits, APSplatUndef, SplatBitSize, 8280 HasAnyUndefs, 0, !Subtarget.isLittleEndian()) || 8281 SplatBitSize > 32) { 8282 // BUILD_VECTOR nodes that are not constant splats of up to 32-bits can be 8283 // lowered to VSX instructions under certain conditions. 8284 // Without VSX, there is no pattern more efficient than expanding the node. 8285 if (Subtarget.hasVSX() && 8286 haveEfficientBuildVectorPattern(BVN, Subtarget.hasDirectMove(), 8287 Subtarget.hasP8Vector())) 8288 return Op; 8289 return SDValue(); 8290 } 8291 8292 unsigned SplatBits = APSplatBits.getZExtValue(); 8293 unsigned SplatUndef = APSplatUndef.getZExtValue(); 8294 unsigned SplatSize = SplatBitSize / 8; 8295 8296 // First, handle single instruction cases. 8297 8298 // All zeros? 8299 if (SplatBits == 0) { 8300 // Canonicalize all zero vectors to be v4i32. 8301 if (Op.getValueType() != MVT::v4i32 || HasAnyUndefs) { 8302 SDValue Z = DAG.getConstant(0, dl, MVT::v4i32); 8303 Op = DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Z); 8304 } 8305 return Op; 8306 } 8307 8308 // We have XXSPLTIB for constant splats one byte wide 8309 if (Subtarget.hasP9Vector() && SplatSize == 1) { 8310 // This is a splat of 1-byte elements with some elements potentially undef. 8311 // Rather than trying to match undef in the SDAG patterns, ensure that all 8312 // elements are the same constant. 8313 if (HasAnyUndefs || ISD::isBuildVectorAllOnes(BVN)) { 8314 SmallVector<SDValue, 16> Ops(16, DAG.getConstant(SplatBits, 8315 dl, MVT::i32)); 8316 SDValue NewBV = DAG.getBuildVector(MVT::v16i8, dl, Ops); 8317 if (Op.getValueType() != MVT::v16i8) 8318 return DAG.getBitcast(Op.getValueType(), NewBV); 8319 return NewBV; 8320 } 8321 8322 // BuildVectorSDNode::isConstantSplat() is actually pretty smart. It'll 8323 // detect that constant splats like v8i16: 0xABAB are really just splats 8324 // of a 1-byte constant. In this case, we need to convert the node to a 8325 // splat of v16i8 and a bitcast. 8326 if (Op.getValueType() != MVT::v16i8) 8327 return DAG.getBitcast(Op.getValueType(), 8328 DAG.getConstant(SplatBits, dl, MVT::v16i8)); 8329 8330 return Op; 8331 } 8332 8333 // If the sign extended value is in the range [-16,15], use VSPLTI[bhw]. 8334 int32_t SextVal= (int32_t(SplatBits << (32-SplatBitSize)) >> 8335 (32-SplatBitSize)); 8336 if (SextVal >= -16 && SextVal <= 15) 8337 return BuildSplatI(SextVal, SplatSize, Op.getValueType(), DAG, dl); 8338 8339 // Two instruction sequences. 8340 8341 // If this value is in the range [-32,30] and is even, use: 8342 // VSPLTI[bhw](val/2) + VSPLTI[bhw](val/2) 8343 // If this value is in the range [17,31] and is odd, use: 8344 // VSPLTI[bhw](val-16) - VSPLTI[bhw](-16) 8345 // If this value is in the range [-31,-17] and is odd, use: 8346 // VSPLTI[bhw](val+16) + VSPLTI[bhw](-16) 8347 // Note the last two are three-instruction sequences. 8348 if (SextVal >= -32 && SextVal <= 31) { 8349 // To avoid having these optimizations undone by constant folding, 8350 // we convert to a pseudo that will be expanded later into one of 8351 // the above forms. 8352 SDValue Elt = DAG.getConstant(SextVal, dl, MVT::i32); 8353 EVT VT = (SplatSize == 1 ? MVT::v16i8 : 8354 (SplatSize == 2 ? MVT::v8i16 : MVT::v4i32)); 8355 SDValue EltSize = DAG.getConstant(SplatSize, dl, MVT::i32); 8356 SDValue RetVal = DAG.getNode(PPCISD::VADD_SPLAT, dl, VT, Elt, EltSize); 8357 if (VT == Op.getValueType()) 8358 return RetVal; 8359 else 8360 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), RetVal); 8361 } 8362 8363 // If this is 0x8000_0000 x 4, turn into vspltisw + vslw. If it is 8364 // 0x7FFF_FFFF x 4, turn it into not(0x8000_0000). This is important 8365 // for fneg/fabs. 8366 if (SplatSize == 4 && SplatBits == (0x7FFFFFFF&~SplatUndef)) { 8367 // Make -1 and vspltisw -1: 8368 SDValue OnesV = BuildSplatI(-1, 4, MVT::v4i32, DAG, dl); 8369 8370 // Make the VSLW intrinsic, computing 0x8000_0000. 8371 SDValue Res = BuildIntrinsicOp(Intrinsic::ppc_altivec_vslw, OnesV, 8372 OnesV, DAG, dl); 8373 8374 // xor by OnesV to invert it. 8375 Res = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Res, OnesV); 8376 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 8377 } 8378 8379 // Check to see if this is a wide variety of vsplti*, binop self cases. 8380 static const signed char SplatCsts[] = { 8381 -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7, 8382 -8, 8, -9, 9, -10, 10, -11, 11, -12, 12, -13, 13, 14, -14, 15, -15, -16 8383 }; 8384 8385 for (unsigned idx = 0; idx < array_lengthof(SplatCsts); ++idx) { 8386 // Indirect through the SplatCsts array so that we favor 'vsplti -1' for 8387 // cases which are ambiguous (e.g. formation of 0x8000_0000). 'vsplti -1' 8388 int i = SplatCsts[idx]; 8389 8390 // Figure out what shift amount will be used by altivec if shifted by i in 8391 // this splat size. 8392 unsigned TypeShiftAmt = i & (SplatBitSize-1); 8393 8394 // vsplti + shl self. 8395 if (SextVal == (int)((unsigned)i << TypeShiftAmt)) { 8396 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); 8397 static const unsigned IIDs[] = { // Intrinsic to use for each size. 8398 Intrinsic::ppc_altivec_vslb, Intrinsic::ppc_altivec_vslh, 0, 8399 Intrinsic::ppc_altivec_vslw 8400 }; 8401 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); 8402 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 8403 } 8404 8405 // vsplti + srl self. 8406 if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) { 8407 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); 8408 static const unsigned IIDs[] = { // Intrinsic to use for each size. 8409 Intrinsic::ppc_altivec_vsrb, Intrinsic::ppc_altivec_vsrh, 0, 8410 Intrinsic::ppc_altivec_vsrw 8411 }; 8412 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); 8413 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 8414 } 8415 8416 // vsplti + sra self. 8417 if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) { 8418 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); 8419 static const unsigned IIDs[] = { // Intrinsic to use for each size. 8420 Intrinsic::ppc_altivec_vsrab, Intrinsic::ppc_altivec_vsrah, 0, 8421 Intrinsic::ppc_altivec_vsraw 8422 }; 8423 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); 8424 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 8425 } 8426 8427 // vsplti + rol self. 8428 if (SextVal == (int)(((unsigned)i << TypeShiftAmt) | 8429 ((unsigned)i >> (SplatBitSize-TypeShiftAmt)))) { 8430 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); 8431 static const unsigned IIDs[] = { // Intrinsic to use for each size. 8432 Intrinsic::ppc_altivec_vrlb, Intrinsic::ppc_altivec_vrlh, 0, 8433 Intrinsic::ppc_altivec_vrlw 8434 }; 8435 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); 8436 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 8437 } 8438 8439 // t = vsplti c, result = vsldoi t, t, 1 8440 if (SextVal == (int)(((unsigned)i << 8) | (i < 0 ? 0xFF : 0))) { 8441 SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl); 8442 unsigned Amt = Subtarget.isLittleEndian() ? 15 : 1; 8443 return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl); 8444 } 8445 // t = vsplti c, result = vsldoi t, t, 2 8446 if (SextVal == (int)(((unsigned)i << 16) | (i < 0 ? 0xFFFF : 0))) { 8447 SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl); 8448 unsigned Amt = Subtarget.isLittleEndian() ? 14 : 2; 8449 return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl); 8450 } 8451 // t = vsplti c, result = vsldoi t, t, 3 8452 if (SextVal == (int)(((unsigned)i << 24) | (i < 0 ? 0xFFFFFF : 0))) { 8453 SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl); 8454 unsigned Amt = Subtarget.isLittleEndian() ? 13 : 3; 8455 return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl); 8456 } 8457 } 8458 8459 return SDValue(); 8460 } 8461 8462 /// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit 8463 /// the specified operations to build the shuffle. 8464 static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS, 8465 SDValue RHS, SelectionDAG &DAG, 8466 const SDLoc &dl) { 8467 unsigned OpNum = (PFEntry >> 26) & 0x0F; 8468 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1); 8469 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1); 8470 8471 enum { 8472 OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3> 8473 OP_VMRGHW, 8474 OP_VMRGLW, 8475 OP_VSPLTISW0, 8476 OP_VSPLTISW1, 8477 OP_VSPLTISW2, 8478 OP_VSPLTISW3, 8479 OP_VSLDOI4, 8480 OP_VSLDOI8, 8481 OP_VSLDOI12 8482 }; 8483 8484 if (OpNum == OP_COPY) { 8485 if (LHSID == (1*9+2)*9+3) return LHS; 8486 assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!"); 8487 return RHS; 8488 } 8489 8490 SDValue OpLHS, OpRHS; 8491 OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl); 8492 OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl); 8493 8494 int ShufIdxs[16]; 8495 switch (OpNum) { 8496 default: llvm_unreachable("Unknown i32 permute!"); 8497 case OP_VMRGHW: 8498 ShufIdxs[ 0] = 0; ShufIdxs[ 1] = 1; ShufIdxs[ 2] = 2; ShufIdxs[ 3] = 3; 8499 ShufIdxs[ 4] = 16; ShufIdxs[ 5] = 17; ShufIdxs[ 6] = 18; ShufIdxs[ 7] = 19; 8500 ShufIdxs[ 8] = 4; ShufIdxs[ 9] = 5; ShufIdxs[10] = 6; ShufIdxs[11] = 7; 8501 ShufIdxs[12] = 20; ShufIdxs[13] = 21; ShufIdxs[14] = 22; ShufIdxs[15] = 23; 8502 break; 8503 case OP_VMRGLW: 8504 ShufIdxs[ 0] = 8; ShufIdxs[ 1] = 9; ShufIdxs[ 2] = 10; ShufIdxs[ 3] = 11; 8505 ShufIdxs[ 4] = 24; ShufIdxs[ 5] = 25; ShufIdxs[ 6] = 26; ShufIdxs[ 7] = 27; 8506 ShufIdxs[ 8] = 12; ShufIdxs[ 9] = 13; ShufIdxs[10] = 14; ShufIdxs[11] = 15; 8507 ShufIdxs[12] = 28; ShufIdxs[13] = 29; ShufIdxs[14] = 30; ShufIdxs[15] = 31; 8508 break; 8509 case OP_VSPLTISW0: 8510 for (unsigned i = 0; i != 16; ++i) 8511 ShufIdxs[i] = (i&3)+0; 8512 break; 8513 case OP_VSPLTISW1: 8514 for (unsigned i = 0; i != 16; ++i) 8515 ShufIdxs[i] = (i&3)+4; 8516 break; 8517 case OP_VSPLTISW2: 8518 for (unsigned i = 0; i != 16; ++i) 8519 ShufIdxs[i] = (i&3)+8; 8520 break; 8521 case OP_VSPLTISW3: 8522 for (unsigned i = 0; i != 16; ++i) 8523 ShufIdxs[i] = (i&3)+12; 8524 break; 8525 case OP_VSLDOI4: 8526 return BuildVSLDOI(OpLHS, OpRHS, 4, OpLHS.getValueType(), DAG, dl); 8527 case OP_VSLDOI8: 8528 return BuildVSLDOI(OpLHS, OpRHS, 8, OpLHS.getValueType(), DAG, dl); 8529 case OP_VSLDOI12: 8530 return BuildVSLDOI(OpLHS, OpRHS, 12, OpLHS.getValueType(), DAG, dl); 8531 } 8532 EVT VT = OpLHS.getValueType(); 8533 OpLHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OpLHS); 8534 OpRHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OpRHS); 8535 SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, OpLHS, OpRHS, ShufIdxs); 8536 return DAG.getNode(ISD::BITCAST, dl, VT, T); 8537 } 8538 8539 /// lowerToVINSERTB - Return the SDValue if this VECTOR_SHUFFLE can be handled 8540 /// by the VINSERTB instruction introduced in ISA 3.0, else just return default 8541 /// SDValue. 8542 SDValue PPCTargetLowering::lowerToVINSERTB(ShuffleVectorSDNode *N, 8543 SelectionDAG &DAG) const { 8544 const unsigned BytesInVector = 16; 8545 bool IsLE = Subtarget.isLittleEndian(); 8546 SDLoc dl(N); 8547 SDValue V1 = N->getOperand(0); 8548 SDValue V2 = N->getOperand(1); 8549 unsigned ShiftElts = 0, InsertAtByte = 0; 8550 bool Swap = false; 8551 8552 // Shifts required to get the byte we want at element 7. 8553 unsigned LittleEndianShifts[] = {8, 7, 6, 5, 4, 3, 2, 1, 8554 0, 15, 14, 13, 12, 11, 10, 9}; 8555 unsigned BigEndianShifts[] = {9, 10, 11, 12, 13, 14, 15, 0, 8556 1, 2, 3, 4, 5, 6, 7, 8}; 8557 8558 ArrayRef<int> Mask = N->getMask(); 8559 int OriginalOrder[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; 8560 8561 // For each mask element, find out if we're just inserting something 8562 // from V2 into V1 or vice versa. 8563 // Possible permutations inserting an element from V2 into V1: 8564 // X, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 8565 // 0, X, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 8566 // ... 8567 // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, X 8568 // Inserting from V1 into V2 will be similar, except mask range will be 8569 // [16,31]. 8570 8571 bool FoundCandidate = false; 8572 // If both vector operands for the shuffle are the same vector, the mask 8573 // will contain only elements from the first one and the second one will be 8574 // undef. 8575 unsigned VINSERTBSrcElem = IsLE ? 8 : 7; 8576 // Go through the mask of half-words to find an element that's being moved 8577 // from one vector to the other. 8578 for (unsigned i = 0; i < BytesInVector; ++i) { 8579 unsigned CurrentElement = Mask[i]; 8580 // If 2nd operand is undefined, we should only look for element 7 in the 8581 // Mask. 8582 if (V2.isUndef() && CurrentElement != VINSERTBSrcElem) 8583 continue; 8584 8585 bool OtherElementsInOrder = true; 8586 // Examine the other elements in the Mask to see if they're in original 8587 // order. 8588 for (unsigned j = 0; j < BytesInVector; ++j) { 8589 if (j == i) 8590 continue; 8591 // If CurrentElement is from V1 [0,15], then we the rest of the Mask to be 8592 // from V2 [16,31] and vice versa. Unless the 2nd operand is undefined, 8593 // in which we always assume we're always picking from the 1st operand. 8594 int MaskOffset = 8595 (!V2.isUndef() && CurrentElement < BytesInVector) ? BytesInVector : 0; 8596 if (Mask[j] != OriginalOrder[j] + MaskOffset) { 8597 OtherElementsInOrder = false; 8598 break; 8599 } 8600 } 8601 // If other elements are in original order, we record the number of shifts 8602 // we need to get the element we want into element 7. Also record which byte 8603 // in the vector we should insert into. 8604 if (OtherElementsInOrder) { 8605 // If 2nd operand is undefined, we assume no shifts and no swapping. 8606 if (V2.isUndef()) { 8607 ShiftElts = 0; 8608 Swap = false; 8609 } else { 8610 // Only need the last 4-bits for shifts because operands will be swapped if CurrentElement is >= 2^4. 8611 ShiftElts = IsLE ? LittleEndianShifts[CurrentElement & 0xF] 8612 : BigEndianShifts[CurrentElement & 0xF]; 8613 Swap = CurrentElement < BytesInVector; 8614 } 8615 InsertAtByte = IsLE ? BytesInVector - (i + 1) : i; 8616 FoundCandidate = true; 8617 break; 8618 } 8619 } 8620 8621 if (!FoundCandidate) 8622 return SDValue(); 8623 8624 // Candidate found, construct the proper SDAG sequence with VINSERTB, 8625 // optionally with VECSHL if shift is required. 8626 if (Swap) 8627 std::swap(V1, V2); 8628 if (V2.isUndef()) 8629 V2 = V1; 8630 if (ShiftElts) { 8631 SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v16i8, V2, V2, 8632 DAG.getConstant(ShiftElts, dl, MVT::i32)); 8633 return DAG.getNode(PPCISD::VECINSERT, dl, MVT::v16i8, V1, Shl, 8634 DAG.getConstant(InsertAtByte, dl, MVT::i32)); 8635 } 8636 return DAG.getNode(PPCISD::VECINSERT, dl, MVT::v16i8, V1, V2, 8637 DAG.getConstant(InsertAtByte, dl, MVT::i32)); 8638 } 8639 8640 /// lowerToVINSERTH - Return the SDValue if this VECTOR_SHUFFLE can be handled 8641 /// by the VINSERTH instruction introduced in ISA 3.0, else just return default 8642 /// SDValue. 8643 SDValue PPCTargetLowering::lowerToVINSERTH(ShuffleVectorSDNode *N, 8644 SelectionDAG &DAG) const { 8645 const unsigned NumHalfWords = 8; 8646 const unsigned BytesInVector = NumHalfWords * 2; 8647 // Check that the shuffle is on half-words. 8648 if (!isNByteElemShuffleMask(N, 2, 1)) 8649 return SDValue(); 8650 8651 bool IsLE = Subtarget.isLittleEndian(); 8652 SDLoc dl(N); 8653 SDValue V1 = N->getOperand(0); 8654 SDValue V2 = N->getOperand(1); 8655 unsigned ShiftElts = 0, InsertAtByte = 0; 8656 bool Swap = false; 8657 8658 // Shifts required to get the half-word we want at element 3. 8659 unsigned LittleEndianShifts[] = {4, 3, 2, 1, 0, 7, 6, 5}; 8660 unsigned BigEndianShifts[] = {5, 6, 7, 0, 1, 2, 3, 4}; 8661 8662 uint32_t Mask = 0; 8663 uint32_t OriginalOrderLow = 0x1234567; 8664 uint32_t OriginalOrderHigh = 0x89ABCDEF; 8665 // Now we look at mask elements 0,2,4,6,8,10,12,14. Pack the mask into a 8666 // 32-bit space, only need 4-bit nibbles per element. 8667 for (unsigned i = 0; i < NumHalfWords; ++i) { 8668 unsigned MaskShift = (NumHalfWords - 1 - i) * 4; 8669 Mask |= ((uint32_t)(N->getMaskElt(i * 2) / 2) << MaskShift); 8670 } 8671 8672 // For each mask element, find out if we're just inserting something 8673 // from V2 into V1 or vice versa. Possible permutations inserting an element 8674 // from V2 into V1: 8675 // X, 1, 2, 3, 4, 5, 6, 7 8676 // 0, X, 2, 3, 4, 5, 6, 7 8677 // 0, 1, X, 3, 4, 5, 6, 7 8678 // 0, 1, 2, X, 4, 5, 6, 7 8679 // 0, 1, 2, 3, X, 5, 6, 7 8680 // 0, 1, 2, 3, 4, X, 6, 7 8681 // 0, 1, 2, 3, 4, 5, X, 7 8682 // 0, 1, 2, 3, 4, 5, 6, X 8683 // Inserting from V1 into V2 will be similar, except mask range will be [8,15]. 8684 8685 bool FoundCandidate = false; 8686 // Go through the mask of half-words to find an element that's being moved 8687 // from one vector to the other. 8688 for (unsigned i = 0; i < NumHalfWords; ++i) { 8689 unsigned MaskShift = (NumHalfWords - 1 - i) * 4; 8690 uint32_t MaskOneElt = (Mask >> MaskShift) & 0xF; 8691 uint32_t MaskOtherElts = ~(0xF << MaskShift); 8692 uint32_t TargetOrder = 0x0; 8693 8694 // If both vector operands for the shuffle are the same vector, the mask 8695 // will contain only elements from the first one and the second one will be 8696 // undef. 8697 if (V2.isUndef()) { 8698 ShiftElts = 0; 8699 unsigned VINSERTHSrcElem = IsLE ? 4 : 3; 8700 TargetOrder = OriginalOrderLow; 8701 Swap = false; 8702 // Skip if not the correct element or mask of other elements don't equal 8703 // to our expected order. 8704 if (MaskOneElt == VINSERTHSrcElem && 8705 (Mask & MaskOtherElts) == (TargetOrder & MaskOtherElts)) { 8706 InsertAtByte = IsLE ? BytesInVector - (i + 1) * 2 : i * 2; 8707 FoundCandidate = true; 8708 break; 8709 } 8710 } else { // If both operands are defined. 8711 // Target order is [8,15] if the current mask is between [0,7]. 8712 TargetOrder = 8713 (MaskOneElt < NumHalfWords) ? OriginalOrderHigh : OriginalOrderLow; 8714 // Skip if mask of other elements don't equal our expected order. 8715 if ((Mask & MaskOtherElts) == (TargetOrder & MaskOtherElts)) { 8716 // We only need the last 3 bits for the number of shifts. 8717 ShiftElts = IsLE ? LittleEndianShifts[MaskOneElt & 0x7] 8718 : BigEndianShifts[MaskOneElt & 0x7]; 8719 InsertAtByte = IsLE ? BytesInVector - (i + 1) * 2 : i * 2; 8720 Swap = MaskOneElt < NumHalfWords; 8721 FoundCandidate = true; 8722 break; 8723 } 8724 } 8725 } 8726 8727 if (!FoundCandidate) 8728 return SDValue(); 8729 8730 // Candidate found, construct the proper SDAG sequence with VINSERTH, 8731 // optionally with VECSHL if shift is required. 8732 if (Swap) 8733 std::swap(V1, V2); 8734 if (V2.isUndef()) 8735 V2 = V1; 8736 SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); 8737 if (ShiftElts) { 8738 // Double ShiftElts because we're left shifting on v16i8 type. 8739 SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v16i8, V2, V2, 8740 DAG.getConstant(2 * ShiftElts, dl, MVT::i32)); 8741 SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, Shl); 8742 SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v8i16, Conv1, Conv2, 8743 DAG.getConstant(InsertAtByte, dl, MVT::i32)); 8744 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins); 8745 } 8746 SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V2); 8747 SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v8i16, Conv1, Conv2, 8748 DAG.getConstant(InsertAtByte, dl, MVT::i32)); 8749 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins); 8750 } 8751 8752 /// LowerVECTOR_SHUFFLE - Return the code we lower for VECTOR_SHUFFLE. If this 8753 /// is a shuffle we can handle in a single instruction, return it. Otherwise, 8754 /// return the code it can be lowered into. Worst case, it can always be 8755 /// lowered into a vperm. 8756 SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, 8757 SelectionDAG &DAG) const { 8758 SDLoc dl(Op); 8759 SDValue V1 = Op.getOperand(0); 8760 SDValue V2 = Op.getOperand(1); 8761 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op); 8762 EVT VT = Op.getValueType(); 8763 bool isLittleEndian = Subtarget.isLittleEndian(); 8764 8765 unsigned ShiftElts, InsertAtByte; 8766 bool Swap = false; 8767 if (Subtarget.hasP9Vector() && 8768 PPC::isXXINSERTWMask(SVOp, ShiftElts, InsertAtByte, Swap, 8769 isLittleEndian)) { 8770 if (Swap) 8771 std::swap(V1, V2); 8772 SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); 8773 SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V2); 8774 if (ShiftElts) { 8775 SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v4i32, Conv2, Conv2, 8776 DAG.getConstant(ShiftElts, dl, MVT::i32)); 8777 SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v4i32, Conv1, Shl, 8778 DAG.getConstant(InsertAtByte, dl, MVT::i32)); 8779 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins); 8780 } 8781 SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v4i32, Conv1, Conv2, 8782 DAG.getConstant(InsertAtByte, dl, MVT::i32)); 8783 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins); 8784 } 8785 8786 if (Subtarget.hasP9Altivec()) { 8787 SDValue NewISDNode; 8788 if ((NewISDNode = lowerToVINSERTH(SVOp, DAG))) 8789 return NewISDNode; 8790 8791 if ((NewISDNode = lowerToVINSERTB(SVOp, DAG))) 8792 return NewISDNode; 8793 } 8794 8795 if (Subtarget.hasVSX() && 8796 PPC::isXXSLDWIShuffleMask(SVOp, ShiftElts, Swap, isLittleEndian)) { 8797 if (Swap) 8798 std::swap(V1, V2); 8799 SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); 8800 SDValue Conv2 = 8801 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V2.isUndef() ? V1 : V2); 8802 8803 SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v4i32, Conv1, Conv2, 8804 DAG.getConstant(ShiftElts, dl, MVT::i32)); 8805 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Shl); 8806 } 8807 8808 if (Subtarget.hasVSX() && 8809 PPC::isXXPERMDIShuffleMask(SVOp, ShiftElts, Swap, isLittleEndian)) { 8810 if (Swap) 8811 std::swap(V1, V2); 8812 SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1); 8813 SDValue Conv2 = 8814 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V2.isUndef() ? V1 : V2); 8815 8816 SDValue PermDI = DAG.getNode(PPCISD::XXPERMDI, dl, MVT::v2i64, Conv1, Conv2, 8817 DAG.getConstant(ShiftElts, dl, MVT::i32)); 8818 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, PermDI); 8819 } 8820 8821 if (Subtarget.hasP9Vector()) { 8822 if (PPC::isXXBRHShuffleMask(SVOp)) { 8823 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); 8824 SDValue ReveHWord = DAG.getNode(PPCISD::XXREVERSE, dl, MVT::v8i16, Conv); 8825 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveHWord); 8826 } else if (PPC::isXXBRWShuffleMask(SVOp)) { 8827 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); 8828 SDValue ReveWord = DAG.getNode(PPCISD::XXREVERSE, dl, MVT::v4i32, Conv); 8829 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveWord); 8830 } else if (PPC::isXXBRDShuffleMask(SVOp)) { 8831 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1); 8832 SDValue ReveDWord = DAG.getNode(PPCISD::XXREVERSE, dl, MVT::v2i64, Conv); 8833 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveDWord); 8834 } else if (PPC::isXXBRQShuffleMask(SVOp)) { 8835 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v1i128, V1); 8836 SDValue ReveQWord = DAG.getNode(PPCISD::XXREVERSE, dl, MVT::v1i128, Conv); 8837 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveQWord); 8838 } 8839 } 8840 8841 if (Subtarget.hasVSX()) { 8842 if (V2.isUndef() && PPC::isSplatShuffleMask(SVOp, 4)) { 8843 int SplatIdx = PPC::getVSPLTImmediate(SVOp, 4, DAG); 8844 8845 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); 8846 SDValue Splat = DAG.getNode(PPCISD::XXSPLT, dl, MVT::v4i32, Conv, 8847 DAG.getConstant(SplatIdx, dl, MVT::i32)); 8848 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Splat); 8849 } 8850 8851 // Left shifts of 8 bytes are actually swaps. Convert accordingly. 8852 if (V2.isUndef() && PPC::isVSLDOIShuffleMask(SVOp, 1, DAG) == 8) { 8853 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, V1); 8854 SDValue Swap = DAG.getNode(PPCISD::SWAP_NO_CHAIN, dl, MVT::v2f64, Conv); 8855 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Swap); 8856 } 8857 } 8858 8859 if (Subtarget.hasQPX()) { 8860 if (VT.getVectorNumElements() != 4) 8861 return SDValue(); 8862 8863 if (V2.isUndef()) V2 = V1; 8864 8865 int AlignIdx = PPC::isQVALIGNIShuffleMask(SVOp); 8866 if (AlignIdx != -1) { 8867 return DAG.getNode(PPCISD::QVALIGNI, dl, VT, V1, V2, 8868 DAG.getConstant(AlignIdx, dl, MVT::i32)); 8869 } else if (SVOp->isSplat()) { 8870 int SplatIdx = SVOp->getSplatIndex(); 8871 if (SplatIdx >= 4) { 8872 std::swap(V1, V2); 8873 SplatIdx -= 4; 8874 } 8875 8876 return DAG.getNode(PPCISD::QVESPLATI, dl, VT, V1, 8877 DAG.getConstant(SplatIdx, dl, MVT::i32)); 8878 } 8879 8880 // Lower this into a qvgpci/qvfperm pair. 8881 8882 // Compute the qvgpci literal 8883 unsigned idx = 0; 8884 for (unsigned i = 0; i < 4; ++i) { 8885 int m = SVOp->getMaskElt(i); 8886 unsigned mm = m >= 0 ? (unsigned) m : i; 8887 idx |= mm << (3-i)*3; 8888 } 8889 8890 SDValue V3 = DAG.getNode(PPCISD::QVGPCI, dl, MVT::v4f64, 8891 DAG.getConstant(idx, dl, MVT::i32)); 8892 return DAG.getNode(PPCISD::QVFPERM, dl, VT, V1, V2, V3); 8893 } 8894 8895 // Cases that are handled by instructions that take permute immediates 8896 // (such as vsplt*) should be left as VECTOR_SHUFFLE nodes so they can be 8897 // selected by the instruction selector. 8898 if (V2.isUndef()) { 8899 if (PPC::isSplatShuffleMask(SVOp, 1) || 8900 PPC::isSplatShuffleMask(SVOp, 2) || 8901 PPC::isSplatShuffleMask(SVOp, 4) || 8902 PPC::isVPKUWUMShuffleMask(SVOp, 1, DAG) || 8903 PPC::isVPKUHUMShuffleMask(SVOp, 1, DAG) || 8904 PPC::isVSLDOIShuffleMask(SVOp, 1, DAG) != -1 || 8905 PPC::isVMRGLShuffleMask(SVOp, 1, 1, DAG) || 8906 PPC::isVMRGLShuffleMask(SVOp, 2, 1, DAG) || 8907 PPC::isVMRGLShuffleMask(SVOp, 4, 1, DAG) || 8908 PPC::isVMRGHShuffleMask(SVOp, 1, 1, DAG) || 8909 PPC::isVMRGHShuffleMask(SVOp, 2, 1, DAG) || 8910 PPC::isVMRGHShuffleMask(SVOp, 4, 1, DAG) || 8911 (Subtarget.hasP8Altivec() && ( 8912 PPC::isVPKUDUMShuffleMask(SVOp, 1, DAG) || 8913 PPC::isVMRGEOShuffleMask(SVOp, true, 1, DAG) || 8914 PPC::isVMRGEOShuffleMask(SVOp, false, 1, DAG)))) { 8915 return Op; 8916 } 8917 } 8918 8919 // Altivec has a variety of "shuffle immediates" that take two vector inputs 8920 // and produce a fixed permutation. If any of these match, do not lower to 8921 // VPERM. 8922 unsigned int ShuffleKind = isLittleEndian ? 2 : 0; 8923 if (PPC::isVPKUWUMShuffleMask(SVOp, ShuffleKind, DAG) || 8924 PPC::isVPKUHUMShuffleMask(SVOp, ShuffleKind, DAG) || 8925 PPC::isVSLDOIShuffleMask(SVOp, ShuffleKind, DAG) != -1 || 8926 PPC::isVMRGLShuffleMask(SVOp, 1, ShuffleKind, DAG) || 8927 PPC::isVMRGLShuffleMask(SVOp, 2, ShuffleKind, DAG) || 8928 PPC::isVMRGLShuffleMask(SVOp, 4, ShuffleKind, DAG) || 8929 PPC::isVMRGHShuffleMask(SVOp, 1, ShuffleKind, DAG) || 8930 PPC::isVMRGHShuffleMask(SVOp, 2, ShuffleKind, DAG) || 8931 PPC::isVMRGHShuffleMask(SVOp, 4, ShuffleKind, DAG) || 8932 (Subtarget.hasP8Altivec() && ( 8933 PPC::isVPKUDUMShuffleMask(SVOp, ShuffleKind, DAG) || 8934 PPC::isVMRGEOShuffleMask(SVOp, true, ShuffleKind, DAG) || 8935 PPC::isVMRGEOShuffleMask(SVOp, false, ShuffleKind, DAG)))) 8936 return Op; 8937 8938 // Check to see if this is a shuffle of 4-byte values. If so, we can use our 8939 // perfect shuffle table to emit an optimal matching sequence. 8940 ArrayRef<int> PermMask = SVOp->getMask(); 8941 8942 unsigned PFIndexes[4]; 8943 bool isFourElementShuffle = true; 8944 for (unsigned i = 0; i != 4 && isFourElementShuffle; ++i) { // Element number 8945 unsigned EltNo = 8; // Start out undef. 8946 for (unsigned j = 0; j != 4; ++j) { // Intra-element byte. 8947 if (PermMask[i*4+j] < 0) 8948 continue; // Undef, ignore it. 8949 8950 unsigned ByteSource = PermMask[i*4+j]; 8951 if ((ByteSource & 3) != j) { 8952 isFourElementShuffle = false; 8953 break; 8954 } 8955 8956 if (EltNo == 8) { 8957 EltNo = ByteSource/4; 8958 } else if (EltNo != ByteSource/4) { 8959 isFourElementShuffle = false; 8960 break; 8961 } 8962 } 8963 PFIndexes[i] = EltNo; 8964 } 8965 8966 // If this shuffle can be expressed as a shuffle of 4-byte elements, use the 8967 // perfect shuffle vector to determine if it is cost effective to do this as 8968 // discrete instructions, or whether we should use a vperm. 8969 // For now, we skip this for little endian until such time as we have a 8970 // little-endian perfect shuffle table. 8971 if (isFourElementShuffle && !isLittleEndian) { 8972 // Compute the index in the perfect shuffle table. 8973 unsigned PFTableIndex = 8974 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3]; 8975 8976 unsigned PFEntry = PerfectShuffleTable[PFTableIndex]; 8977 unsigned Cost = (PFEntry >> 30); 8978 8979 // Determining when to avoid vperm is tricky. Many things affect the cost 8980 // of vperm, particularly how many times the perm mask needs to be computed. 8981 // For example, if the perm mask can be hoisted out of a loop or is already 8982 // used (perhaps because there are multiple permutes with the same shuffle 8983 // mask?) the vperm has a cost of 1. OTOH, hoisting the permute mask out of 8984 // the loop requires an extra register. 8985 // 8986 // As a compromise, we only emit discrete instructions if the shuffle can be 8987 // generated in 3 or fewer operations. When we have loop information 8988 // available, if this block is within a loop, we should avoid using vperm 8989 // for 3-operation perms and use a constant pool load instead. 8990 if (Cost < 3) 8991 return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl); 8992 } 8993 8994 // Lower this to a VPERM(V1, V2, V3) expression, where V3 is a constant 8995 // vector that will get spilled to the constant pool. 8996 if (V2.isUndef()) V2 = V1; 8997 8998 // The SHUFFLE_VECTOR mask is almost exactly what we want for vperm, except 8999 // that it is in input element units, not in bytes. Convert now. 9000 9001 // For little endian, the order of the input vectors is reversed, and 9002 // the permutation mask is complemented with respect to 31. This is 9003 // necessary to produce proper semantics with the big-endian-biased vperm 9004 // instruction. 9005 EVT EltVT = V1.getValueType().getVectorElementType(); 9006 unsigned BytesPerElement = EltVT.getSizeInBits()/8; 9007 9008 SmallVector<SDValue, 16> ResultMask; 9009 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) { 9010 unsigned SrcElt = PermMask[i] < 0 ? 0 : PermMask[i]; 9011 9012 for (unsigned j = 0; j != BytesPerElement; ++j) 9013 if (isLittleEndian) 9014 ResultMask.push_back(DAG.getConstant(31 - (SrcElt*BytesPerElement + j), 9015 dl, MVT::i32)); 9016 else 9017 ResultMask.push_back(DAG.getConstant(SrcElt*BytesPerElement + j, dl, 9018 MVT::i32)); 9019 } 9020 9021 SDValue VPermMask = DAG.getBuildVector(MVT::v16i8, dl, ResultMask); 9022 if (isLittleEndian) 9023 return DAG.getNode(PPCISD::VPERM, dl, V1.getValueType(), 9024 V2, V1, VPermMask); 9025 else 9026 return DAG.getNode(PPCISD::VPERM, dl, V1.getValueType(), 9027 V1, V2, VPermMask); 9028 } 9029 9030 /// getVectorCompareInfo - Given an intrinsic, return false if it is not a 9031 /// vector comparison. If it is, return true and fill in Opc/isDot with 9032 /// information about the intrinsic. 9033 static bool getVectorCompareInfo(SDValue Intrin, int &CompareOpc, 9034 bool &isDot, const PPCSubtarget &Subtarget) { 9035 unsigned IntrinsicID = 9036 cast<ConstantSDNode>(Intrin.getOperand(0))->getZExtValue(); 9037 CompareOpc = -1; 9038 isDot = false; 9039 switch (IntrinsicID) { 9040 default: 9041 return false; 9042 // Comparison predicates. 9043 case Intrinsic::ppc_altivec_vcmpbfp_p: 9044 CompareOpc = 966; 9045 isDot = true; 9046 break; 9047 case Intrinsic::ppc_altivec_vcmpeqfp_p: 9048 CompareOpc = 198; 9049 isDot = true; 9050 break; 9051 case Intrinsic::ppc_altivec_vcmpequb_p: 9052 CompareOpc = 6; 9053 isDot = true; 9054 break; 9055 case Intrinsic::ppc_altivec_vcmpequh_p: 9056 CompareOpc = 70; 9057 isDot = true; 9058 break; 9059 case Intrinsic::ppc_altivec_vcmpequw_p: 9060 CompareOpc = 134; 9061 isDot = true; 9062 break; 9063 case Intrinsic::ppc_altivec_vcmpequd_p: 9064 if (Subtarget.hasP8Altivec()) { 9065 CompareOpc = 199; 9066 isDot = true; 9067 } else 9068 return false; 9069 break; 9070 case Intrinsic::ppc_altivec_vcmpneb_p: 9071 case Intrinsic::ppc_altivec_vcmpneh_p: 9072 case Intrinsic::ppc_altivec_vcmpnew_p: 9073 case Intrinsic::ppc_altivec_vcmpnezb_p: 9074 case Intrinsic::ppc_altivec_vcmpnezh_p: 9075 case Intrinsic::ppc_altivec_vcmpnezw_p: 9076 if (Subtarget.hasP9Altivec()) { 9077 switch (IntrinsicID) { 9078 default: 9079 llvm_unreachable("Unknown comparison intrinsic."); 9080 case Intrinsic::ppc_altivec_vcmpneb_p: 9081 CompareOpc = 7; 9082 break; 9083 case Intrinsic::ppc_altivec_vcmpneh_p: 9084 CompareOpc = 71; 9085 break; 9086 case Intrinsic::ppc_altivec_vcmpnew_p: 9087 CompareOpc = 135; 9088 break; 9089 case Intrinsic::ppc_altivec_vcmpnezb_p: 9090 CompareOpc = 263; 9091 break; 9092 case Intrinsic::ppc_altivec_vcmpnezh_p: 9093 CompareOpc = 327; 9094 break; 9095 case Intrinsic::ppc_altivec_vcmpnezw_p: 9096 CompareOpc = 391; 9097 break; 9098 } 9099 isDot = true; 9100 } else 9101 return false; 9102 break; 9103 case Intrinsic::ppc_altivec_vcmpgefp_p: 9104 CompareOpc = 454; 9105 isDot = true; 9106 break; 9107 case Intrinsic::ppc_altivec_vcmpgtfp_p: 9108 CompareOpc = 710; 9109 isDot = true; 9110 break; 9111 case Intrinsic::ppc_altivec_vcmpgtsb_p: 9112 CompareOpc = 774; 9113 isDot = true; 9114 break; 9115 case Intrinsic::ppc_altivec_vcmpgtsh_p: 9116 CompareOpc = 838; 9117 isDot = true; 9118 break; 9119 case Intrinsic::ppc_altivec_vcmpgtsw_p: 9120 CompareOpc = 902; 9121 isDot = true; 9122 break; 9123 case Intrinsic::ppc_altivec_vcmpgtsd_p: 9124 if (Subtarget.hasP8Altivec()) { 9125 CompareOpc = 967; 9126 isDot = true; 9127 } else 9128 return false; 9129 break; 9130 case Intrinsic::ppc_altivec_vcmpgtub_p: 9131 CompareOpc = 518; 9132 isDot = true; 9133 break; 9134 case Intrinsic::ppc_altivec_vcmpgtuh_p: 9135 CompareOpc = 582; 9136 isDot = true; 9137 break; 9138 case Intrinsic::ppc_altivec_vcmpgtuw_p: 9139 CompareOpc = 646; 9140 isDot = true; 9141 break; 9142 case Intrinsic::ppc_altivec_vcmpgtud_p: 9143 if (Subtarget.hasP8Altivec()) { 9144 CompareOpc = 711; 9145 isDot = true; 9146 } else 9147 return false; 9148 break; 9149 9150 // VSX predicate comparisons use the same infrastructure 9151 case Intrinsic::ppc_vsx_xvcmpeqdp_p: 9152 case Intrinsic::ppc_vsx_xvcmpgedp_p: 9153 case Intrinsic::ppc_vsx_xvcmpgtdp_p: 9154 case Intrinsic::ppc_vsx_xvcmpeqsp_p: 9155 case Intrinsic::ppc_vsx_xvcmpgesp_p: 9156 case Intrinsic::ppc_vsx_xvcmpgtsp_p: 9157 if (Subtarget.hasVSX()) { 9158 switch (IntrinsicID) { 9159 case Intrinsic::ppc_vsx_xvcmpeqdp_p: 9160 CompareOpc = 99; 9161 break; 9162 case Intrinsic::ppc_vsx_xvcmpgedp_p: 9163 CompareOpc = 115; 9164 break; 9165 case Intrinsic::ppc_vsx_xvcmpgtdp_p: 9166 CompareOpc = 107; 9167 break; 9168 case Intrinsic::ppc_vsx_xvcmpeqsp_p: 9169 CompareOpc = 67; 9170 break; 9171 case Intrinsic::ppc_vsx_xvcmpgesp_p: 9172 CompareOpc = 83; 9173 break; 9174 case Intrinsic::ppc_vsx_xvcmpgtsp_p: 9175 CompareOpc = 75; 9176 break; 9177 } 9178 isDot = true; 9179 } else 9180 return false; 9181 break; 9182 9183 // Normal Comparisons. 9184 case Intrinsic::ppc_altivec_vcmpbfp: 9185 CompareOpc = 966; 9186 break; 9187 case Intrinsic::ppc_altivec_vcmpeqfp: 9188 CompareOpc = 198; 9189 break; 9190 case Intrinsic::ppc_altivec_vcmpequb: 9191 CompareOpc = 6; 9192 break; 9193 case Intrinsic::ppc_altivec_vcmpequh: 9194 CompareOpc = 70; 9195 break; 9196 case Intrinsic::ppc_altivec_vcmpequw: 9197 CompareOpc = 134; 9198 break; 9199 case Intrinsic::ppc_altivec_vcmpequd: 9200 if (Subtarget.hasP8Altivec()) 9201 CompareOpc = 199; 9202 else 9203 return false; 9204 break; 9205 case Intrinsic::ppc_altivec_vcmpneb: 9206 case Intrinsic::ppc_altivec_vcmpneh: 9207 case Intrinsic::ppc_altivec_vcmpnew: 9208 case Intrinsic::ppc_altivec_vcmpnezb: 9209 case Intrinsic::ppc_altivec_vcmpnezh: 9210 case Intrinsic::ppc_altivec_vcmpnezw: 9211 if (Subtarget.hasP9Altivec()) 9212 switch (IntrinsicID) { 9213 default: 9214 llvm_unreachable("Unknown comparison intrinsic."); 9215 case Intrinsic::ppc_altivec_vcmpneb: 9216 CompareOpc = 7; 9217 break; 9218 case Intrinsic::ppc_altivec_vcmpneh: 9219 CompareOpc = 71; 9220 break; 9221 case Intrinsic::ppc_altivec_vcmpnew: 9222 CompareOpc = 135; 9223 break; 9224 case Intrinsic::ppc_altivec_vcmpnezb: 9225 CompareOpc = 263; 9226 break; 9227 case Intrinsic::ppc_altivec_vcmpnezh: 9228 CompareOpc = 327; 9229 break; 9230 case Intrinsic::ppc_altivec_vcmpnezw: 9231 CompareOpc = 391; 9232 break; 9233 } 9234 else 9235 return false; 9236 break; 9237 case Intrinsic::ppc_altivec_vcmpgefp: 9238 CompareOpc = 454; 9239 break; 9240 case Intrinsic::ppc_altivec_vcmpgtfp: 9241 CompareOpc = 710; 9242 break; 9243 case Intrinsic::ppc_altivec_vcmpgtsb: 9244 CompareOpc = 774; 9245 break; 9246 case Intrinsic::ppc_altivec_vcmpgtsh: 9247 CompareOpc = 838; 9248 break; 9249 case Intrinsic::ppc_altivec_vcmpgtsw: 9250 CompareOpc = 902; 9251 break; 9252 case Intrinsic::ppc_altivec_vcmpgtsd: 9253 if (Subtarget.hasP8Altivec()) 9254 CompareOpc = 967; 9255 else 9256 return false; 9257 break; 9258 case Intrinsic::ppc_altivec_vcmpgtub: 9259 CompareOpc = 518; 9260 break; 9261 case Intrinsic::ppc_altivec_vcmpgtuh: 9262 CompareOpc = 582; 9263 break; 9264 case Intrinsic::ppc_altivec_vcmpgtuw: 9265 CompareOpc = 646; 9266 break; 9267 case Intrinsic::ppc_altivec_vcmpgtud: 9268 if (Subtarget.hasP8Altivec()) 9269 CompareOpc = 711; 9270 else 9271 return false; 9272 break; 9273 } 9274 return true; 9275 } 9276 9277 /// LowerINTRINSIC_WO_CHAIN - If this is an intrinsic that we want to custom 9278 /// lower, do it, otherwise return null. 9279 SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, 9280 SelectionDAG &DAG) const { 9281 unsigned IntrinsicID = 9282 cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 9283 9284 SDLoc dl(Op); 9285 9286 if (IntrinsicID == Intrinsic::thread_pointer) { 9287 // Reads the thread pointer register, used for __builtin_thread_pointer. 9288 if (Subtarget.isPPC64()) 9289 return DAG.getRegister(PPC::X13, MVT::i64); 9290 return DAG.getRegister(PPC::R2, MVT::i32); 9291 } 9292 9293 // If this is a lowered altivec predicate compare, CompareOpc is set to the 9294 // opcode number of the comparison. 9295 int CompareOpc; 9296 bool isDot; 9297 if (!getVectorCompareInfo(Op, CompareOpc, isDot, Subtarget)) 9298 return SDValue(); // Don't custom lower most intrinsics. 9299 9300 // If this is a non-dot comparison, make the VCMP node and we are done. 9301 if (!isDot) { 9302 SDValue Tmp = DAG.getNode(PPCISD::VCMP, dl, Op.getOperand(2).getValueType(), 9303 Op.getOperand(1), Op.getOperand(2), 9304 DAG.getConstant(CompareOpc, dl, MVT::i32)); 9305 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Tmp); 9306 } 9307 9308 // Create the PPCISD altivec 'dot' comparison node. 9309 SDValue Ops[] = { 9310 Op.getOperand(2), // LHS 9311 Op.getOperand(3), // RHS 9312 DAG.getConstant(CompareOpc, dl, MVT::i32) 9313 }; 9314 EVT VTs[] = { Op.getOperand(2).getValueType(), MVT::Glue }; 9315 SDValue CompNode = DAG.getNode(PPCISD::VCMPo, dl, VTs, Ops); 9316 9317 // Now that we have the comparison, emit a copy from the CR to a GPR. 9318 // This is flagged to the above dot comparison. 9319 SDValue Flags = DAG.getNode(PPCISD::MFOCRF, dl, MVT::i32, 9320 DAG.getRegister(PPC::CR6, MVT::i32), 9321 CompNode.getValue(1)); 9322 9323 // Unpack the result based on how the target uses it. 9324 unsigned BitNo; // Bit # of CR6. 9325 bool InvertBit; // Invert result? 9326 switch (cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue()) { 9327 default: // Can't happen, don't crash on invalid number though. 9328 case 0: // Return the value of the EQ bit of CR6. 9329 BitNo = 0; InvertBit = false; 9330 break; 9331 case 1: // Return the inverted value of the EQ bit of CR6. 9332 BitNo = 0; InvertBit = true; 9333 break; 9334 case 2: // Return the value of the LT bit of CR6. 9335 BitNo = 2; InvertBit = false; 9336 break; 9337 case 3: // Return the inverted value of the LT bit of CR6. 9338 BitNo = 2; InvertBit = true; 9339 break; 9340 } 9341 9342 // Shift the bit into the low position. 9343 Flags = DAG.getNode(ISD::SRL, dl, MVT::i32, Flags, 9344 DAG.getConstant(8 - (3 - BitNo), dl, MVT::i32)); 9345 // Isolate the bit. 9346 Flags = DAG.getNode(ISD::AND, dl, MVT::i32, Flags, 9347 DAG.getConstant(1, dl, MVT::i32)); 9348 9349 // If we are supposed to, toggle the bit. 9350 if (InvertBit) 9351 Flags = DAG.getNode(ISD::XOR, dl, MVT::i32, Flags, 9352 DAG.getConstant(1, dl, MVT::i32)); 9353 return Flags; 9354 } 9355 9356 SDValue PPCTargetLowering::LowerINTRINSIC_VOID(SDValue Op, 9357 SelectionDAG &DAG) const { 9358 // SelectionDAGBuilder::visitTargetIntrinsic may insert one extra chain to 9359 // the beginning of the argument list. 9360 int ArgStart = isa<ConstantSDNode>(Op.getOperand(0)) ? 0 : 1; 9361 SDLoc DL(Op); 9362 switch (cast<ConstantSDNode>(Op.getOperand(ArgStart))->getZExtValue()) { 9363 case Intrinsic::ppc_cfence: { 9364 assert(ArgStart == 1 && "llvm.ppc.cfence must carry a chain argument."); 9365 assert(Subtarget.isPPC64() && "Only 64-bit is supported for now."); 9366 return SDValue(DAG.getMachineNode(PPC::CFENCE8, DL, MVT::Other, 9367 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, 9368 Op.getOperand(ArgStart + 1)), 9369 Op.getOperand(0)), 9370 0); 9371 } 9372 default: 9373 break; 9374 } 9375 return SDValue(); 9376 } 9377 9378 SDValue PPCTargetLowering::LowerREM(SDValue Op, SelectionDAG &DAG) const { 9379 // Check for a DIV with the same operands as this REM. 9380 for (auto UI : Op.getOperand(1)->uses()) { 9381 if ((Op.getOpcode() == ISD::SREM && UI->getOpcode() == ISD::SDIV) || 9382 (Op.getOpcode() == ISD::UREM && UI->getOpcode() == ISD::UDIV)) 9383 if (UI->getOperand(0) == Op.getOperand(0) && 9384 UI->getOperand(1) == Op.getOperand(1)) 9385 return SDValue(); 9386 } 9387 return Op; 9388 } 9389 9390 // Lower scalar BSWAP64 to xxbrd. 9391 SDValue PPCTargetLowering::LowerBSWAP(SDValue Op, SelectionDAG &DAG) const { 9392 SDLoc dl(Op); 9393 // MTVSRDD 9394 Op = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2i64, Op.getOperand(0), 9395 Op.getOperand(0)); 9396 // XXBRD 9397 Op = DAG.getNode(PPCISD::XXREVERSE, dl, MVT::v2i64, Op); 9398 // MFVSRD 9399 int VectorIndex = 0; 9400 if (Subtarget.isLittleEndian()) 9401 VectorIndex = 1; 9402 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Op, 9403 DAG.getTargetConstant(VectorIndex, dl, MVT::i32)); 9404 return Op; 9405 } 9406 9407 // ATOMIC_CMP_SWAP for i8/i16 needs to zero-extend its input since it will be 9408 // compared to a value that is atomically loaded (atomic loads zero-extend). 9409 SDValue PPCTargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op, 9410 SelectionDAG &DAG) const { 9411 assert(Op.getOpcode() == ISD::ATOMIC_CMP_SWAP && 9412 "Expecting an atomic compare-and-swap here."); 9413 SDLoc dl(Op); 9414 auto *AtomicNode = cast<AtomicSDNode>(Op.getNode()); 9415 EVT MemVT = AtomicNode->getMemoryVT(); 9416 if (MemVT.getSizeInBits() >= 32) 9417 return Op; 9418 9419 SDValue CmpOp = Op.getOperand(2); 9420 // If this is already correctly zero-extended, leave it alone. 9421 auto HighBits = APInt::getHighBitsSet(32, 32 - MemVT.getSizeInBits()); 9422 if (DAG.MaskedValueIsZero(CmpOp, HighBits)) 9423 return Op; 9424 9425 // Clear the high bits of the compare operand. 9426 unsigned MaskVal = (1 << MemVT.getSizeInBits()) - 1; 9427 SDValue NewCmpOp = 9428 DAG.getNode(ISD::AND, dl, MVT::i32, CmpOp, 9429 DAG.getConstant(MaskVal, dl, MVT::i32)); 9430 9431 // Replace the existing compare operand with the properly zero-extended one. 9432 SmallVector<SDValue, 4> Ops; 9433 for (int i = 0, e = AtomicNode->getNumOperands(); i < e; i++) 9434 Ops.push_back(AtomicNode->getOperand(i)); 9435 Ops[2] = NewCmpOp; 9436 MachineMemOperand *MMO = AtomicNode->getMemOperand(); 9437 SDVTList Tys = DAG.getVTList(MVT::i32, MVT::Other); 9438 auto NodeTy = 9439 (MemVT == MVT::i8) ? PPCISD::ATOMIC_CMP_SWAP_8 : PPCISD::ATOMIC_CMP_SWAP_16; 9440 return DAG.getMemIntrinsicNode(NodeTy, dl, Tys, Ops, MemVT, MMO); 9441 } 9442 9443 SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op, 9444 SelectionDAG &DAG) const { 9445 SDLoc dl(Op); 9446 // Create a stack slot that is 16-byte aligned. 9447 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); 9448 int FrameIdx = MFI.CreateStackObject(16, 16, false); 9449 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 9450 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 9451 9452 // Store the input value into Value#0 of the stack slot. 9453 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), FIdx, 9454 MachinePointerInfo()); 9455 // Load it out. 9456 return DAG.getLoad(Op.getValueType(), dl, Store, FIdx, MachinePointerInfo()); 9457 } 9458 9459 SDValue PPCTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, 9460 SelectionDAG &DAG) const { 9461 assert(Op.getOpcode() == ISD::INSERT_VECTOR_ELT && 9462 "Should only be called for ISD::INSERT_VECTOR_ELT"); 9463 9464 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(2)); 9465 // We have legal lowering for constant indices but not for variable ones. 9466 if (!C) 9467 return SDValue(); 9468 9469 EVT VT = Op.getValueType(); 9470 SDLoc dl(Op); 9471 SDValue V1 = Op.getOperand(0); 9472 SDValue V2 = Op.getOperand(1); 9473 // We can use MTVSRZ + VECINSERT for v8i16 and v16i8 types. 9474 if (VT == MVT::v8i16 || VT == MVT::v16i8) { 9475 SDValue Mtvsrz = DAG.getNode(PPCISD::MTVSRZ, dl, VT, V2); 9476 unsigned BytesInEachElement = VT.getVectorElementType().getSizeInBits() / 8; 9477 unsigned InsertAtElement = C->getZExtValue(); 9478 unsigned InsertAtByte = InsertAtElement * BytesInEachElement; 9479 if (Subtarget.isLittleEndian()) { 9480 InsertAtByte = (16 - BytesInEachElement) - InsertAtByte; 9481 } 9482 return DAG.getNode(PPCISD::VECINSERT, dl, VT, V1, Mtvsrz, 9483 DAG.getConstant(InsertAtByte, dl, MVT::i32)); 9484 } 9485 return Op; 9486 } 9487 9488 SDValue PPCTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, 9489 SelectionDAG &DAG) const { 9490 SDLoc dl(Op); 9491 SDNode *N = Op.getNode(); 9492 9493 assert(N->getOperand(0).getValueType() == MVT::v4i1 && 9494 "Unknown extract_vector_elt type"); 9495 9496 SDValue Value = N->getOperand(0); 9497 9498 // The first part of this is like the store lowering except that we don't 9499 // need to track the chain. 9500 9501 // The values are now known to be -1 (false) or 1 (true). To convert this 9502 // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5). 9503 // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5 9504 Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value); 9505 9506 // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to 9507 // understand how to form the extending load. 9508 SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64); 9509 9510 Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs); 9511 9512 // Now convert to an integer and store. 9513 Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64, 9514 DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32), 9515 Value); 9516 9517 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); 9518 int FrameIdx = MFI.CreateStackObject(16, 16, false); 9519 MachinePointerInfo PtrInfo = 9520 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 9521 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 9522 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 9523 9524 SDValue StoreChain = DAG.getEntryNode(); 9525 SDValue Ops[] = {StoreChain, 9526 DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32), 9527 Value, FIdx}; 9528 SDVTList VTs = DAG.getVTList(/*chain*/ MVT::Other); 9529 9530 StoreChain = DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, 9531 dl, VTs, Ops, MVT::v4i32, PtrInfo); 9532 9533 // Extract the value requested. 9534 unsigned Offset = 4*cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 9535 SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType()); 9536 Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx); 9537 9538 SDValue IntVal = 9539 DAG.getLoad(MVT::i32, dl, StoreChain, Idx, PtrInfo.getWithOffset(Offset)); 9540 9541 if (!Subtarget.useCRBits()) 9542 return IntVal; 9543 9544 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, IntVal); 9545 } 9546 9547 /// Lowering for QPX v4i1 loads 9548 SDValue PPCTargetLowering::LowerVectorLoad(SDValue Op, 9549 SelectionDAG &DAG) const { 9550 SDLoc dl(Op); 9551 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode()); 9552 SDValue LoadChain = LN->getChain(); 9553 SDValue BasePtr = LN->getBasePtr(); 9554 9555 if (Op.getValueType() == MVT::v4f64 || 9556 Op.getValueType() == MVT::v4f32) { 9557 EVT MemVT = LN->getMemoryVT(); 9558 unsigned Alignment = LN->getAlignment(); 9559 9560 // If this load is properly aligned, then it is legal. 9561 if (Alignment >= MemVT.getStoreSize()) 9562 return Op; 9563 9564 EVT ScalarVT = Op.getValueType().getScalarType(), 9565 ScalarMemVT = MemVT.getScalarType(); 9566 unsigned Stride = ScalarMemVT.getStoreSize(); 9567 9568 SDValue Vals[4], LoadChains[4]; 9569 for (unsigned Idx = 0; Idx < 4; ++Idx) { 9570 SDValue Load; 9571 if (ScalarVT != ScalarMemVT) 9572 Load = DAG.getExtLoad(LN->getExtensionType(), dl, ScalarVT, LoadChain, 9573 BasePtr, 9574 LN->getPointerInfo().getWithOffset(Idx * Stride), 9575 ScalarMemVT, MinAlign(Alignment, Idx * Stride), 9576 LN->getMemOperand()->getFlags(), LN->getAAInfo()); 9577 else 9578 Load = DAG.getLoad(ScalarVT, dl, LoadChain, BasePtr, 9579 LN->getPointerInfo().getWithOffset(Idx * Stride), 9580 MinAlign(Alignment, Idx * Stride), 9581 LN->getMemOperand()->getFlags(), LN->getAAInfo()); 9582 9583 if (Idx == 0 && LN->isIndexed()) { 9584 assert(LN->getAddressingMode() == ISD::PRE_INC && 9585 "Unknown addressing mode on vector load"); 9586 Load = DAG.getIndexedLoad(Load, dl, BasePtr, LN->getOffset(), 9587 LN->getAddressingMode()); 9588 } 9589 9590 Vals[Idx] = Load; 9591 LoadChains[Idx] = Load.getValue(1); 9592 9593 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 9594 DAG.getConstant(Stride, dl, 9595 BasePtr.getValueType())); 9596 } 9597 9598 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); 9599 SDValue Value = DAG.getBuildVector(Op.getValueType(), dl, Vals); 9600 9601 if (LN->isIndexed()) { 9602 SDValue RetOps[] = { Value, Vals[0].getValue(1), TF }; 9603 return DAG.getMergeValues(RetOps, dl); 9604 } 9605 9606 SDValue RetOps[] = { Value, TF }; 9607 return DAG.getMergeValues(RetOps, dl); 9608 } 9609 9610 assert(Op.getValueType() == MVT::v4i1 && "Unknown load to lower"); 9611 assert(LN->isUnindexed() && "Indexed v4i1 loads are not supported"); 9612 9613 // To lower v4i1 from a byte array, we load the byte elements of the 9614 // vector and then reuse the BUILD_VECTOR logic. 9615 9616 SDValue VectElmts[4], VectElmtChains[4]; 9617 for (unsigned i = 0; i < 4; ++i) { 9618 SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType()); 9619 Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx); 9620 9621 VectElmts[i] = DAG.getExtLoad( 9622 ISD::EXTLOAD, dl, MVT::i32, LoadChain, Idx, 9623 LN->getPointerInfo().getWithOffset(i), MVT::i8, 9624 /* Alignment = */ 1, LN->getMemOperand()->getFlags(), LN->getAAInfo()); 9625 VectElmtChains[i] = VectElmts[i].getValue(1); 9626 } 9627 9628 LoadChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, VectElmtChains); 9629 SDValue Value = DAG.getBuildVector(MVT::v4i1, dl, VectElmts); 9630 9631 SDValue RVals[] = { Value, LoadChain }; 9632 return DAG.getMergeValues(RVals, dl); 9633 } 9634 9635 /// Lowering for QPX v4i1 stores 9636 SDValue PPCTargetLowering::LowerVectorStore(SDValue Op, 9637 SelectionDAG &DAG) const { 9638 SDLoc dl(Op); 9639 StoreSDNode *SN = cast<StoreSDNode>(Op.getNode()); 9640 SDValue StoreChain = SN->getChain(); 9641 SDValue BasePtr = SN->getBasePtr(); 9642 SDValue Value = SN->getValue(); 9643 9644 if (Value.getValueType() == MVT::v4f64 || 9645 Value.getValueType() == MVT::v4f32) { 9646 EVT MemVT = SN->getMemoryVT(); 9647 unsigned Alignment = SN->getAlignment(); 9648 9649 // If this store is properly aligned, then it is legal. 9650 if (Alignment >= MemVT.getStoreSize()) 9651 return Op; 9652 9653 EVT ScalarVT = Value.getValueType().getScalarType(), 9654 ScalarMemVT = MemVT.getScalarType(); 9655 unsigned Stride = ScalarMemVT.getStoreSize(); 9656 9657 SDValue Stores[4]; 9658 for (unsigned Idx = 0; Idx < 4; ++Idx) { 9659 SDValue Ex = DAG.getNode( 9660 ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, Value, 9661 DAG.getConstant(Idx, dl, getVectorIdxTy(DAG.getDataLayout()))); 9662 SDValue Store; 9663 if (ScalarVT != ScalarMemVT) 9664 Store = 9665 DAG.getTruncStore(StoreChain, dl, Ex, BasePtr, 9666 SN->getPointerInfo().getWithOffset(Idx * Stride), 9667 ScalarMemVT, MinAlign(Alignment, Idx * Stride), 9668 SN->getMemOperand()->getFlags(), SN->getAAInfo()); 9669 else 9670 Store = DAG.getStore(StoreChain, dl, Ex, BasePtr, 9671 SN->getPointerInfo().getWithOffset(Idx * Stride), 9672 MinAlign(Alignment, Idx * Stride), 9673 SN->getMemOperand()->getFlags(), SN->getAAInfo()); 9674 9675 if (Idx == 0 && SN->isIndexed()) { 9676 assert(SN->getAddressingMode() == ISD::PRE_INC && 9677 "Unknown addressing mode on vector store"); 9678 Store = DAG.getIndexedStore(Store, dl, BasePtr, SN->getOffset(), 9679 SN->getAddressingMode()); 9680 } 9681 9682 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 9683 DAG.getConstant(Stride, dl, 9684 BasePtr.getValueType())); 9685 Stores[Idx] = Store; 9686 } 9687 9688 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); 9689 9690 if (SN->isIndexed()) { 9691 SDValue RetOps[] = { TF, Stores[0].getValue(1) }; 9692 return DAG.getMergeValues(RetOps, dl); 9693 } 9694 9695 return TF; 9696 } 9697 9698 assert(SN->isUnindexed() && "Indexed v4i1 stores are not supported"); 9699 assert(Value.getValueType() == MVT::v4i1 && "Unknown store to lower"); 9700 9701 // The values are now known to be -1 (false) or 1 (true). To convert this 9702 // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5). 9703 // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5 9704 Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value); 9705 9706 // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to 9707 // understand how to form the extending load. 9708 SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64); 9709 9710 Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs); 9711 9712 // Now convert to an integer and store. 9713 Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64, 9714 DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32), 9715 Value); 9716 9717 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); 9718 int FrameIdx = MFI.CreateStackObject(16, 16, false); 9719 MachinePointerInfo PtrInfo = 9720 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 9721 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 9722 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 9723 9724 SDValue Ops[] = {StoreChain, 9725 DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32), 9726 Value, FIdx}; 9727 SDVTList VTs = DAG.getVTList(/*chain*/ MVT::Other); 9728 9729 StoreChain = DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, 9730 dl, VTs, Ops, MVT::v4i32, PtrInfo); 9731 9732 // Move data into the byte array. 9733 SDValue Loads[4], LoadChains[4]; 9734 for (unsigned i = 0; i < 4; ++i) { 9735 unsigned Offset = 4*i; 9736 SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType()); 9737 Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx); 9738 9739 Loads[i] = DAG.getLoad(MVT::i32, dl, StoreChain, Idx, 9740 PtrInfo.getWithOffset(Offset)); 9741 LoadChains[i] = Loads[i].getValue(1); 9742 } 9743 9744 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); 9745 9746 SDValue Stores[4]; 9747 for (unsigned i = 0; i < 4; ++i) { 9748 SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType()); 9749 Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx); 9750 9751 Stores[i] = DAG.getTruncStore( 9752 StoreChain, dl, Loads[i], Idx, SN->getPointerInfo().getWithOffset(i), 9753 MVT::i8, /* Alignment = */ 1, SN->getMemOperand()->getFlags(), 9754 SN->getAAInfo()); 9755 } 9756 9757 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); 9758 9759 return StoreChain; 9760 } 9761 9762 SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const { 9763 SDLoc dl(Op); 9764 if (Op.getValueType() == MVT::v4i32) { 9765 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); 9766 9767 SDValue Zero = BuildSplatI( 0, 1, MVT::v4i32, DAG, dl); 9768 SDValue Neg16 = BuildSplatI(-16, 4, MVT::v4i32, DAG, dl);//+16 as shift amt. 9769 9770 SDValue RHSSwap = // = vrlw RHS, 16 9771 BuildIntrinsicOp(Intrinsic::ppc_altivec_vrlw, RHS, Neg16, DAG, dl); 9772 9773 // Shrinkify inputs to v8i16. 9774 LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, LHS); 9775 RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, RHS); 9776 RHSSwap = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, RHSSwap); 9777 9778 // Low parts multiplied together, generating 32-bit results (we ignore the 9779 // top parts). 9780 SDValue LoProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmulouh, 9781 LHS, RHS, DAG, dl, MVT::v4i32); 9782 9783 SDValue HiProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmsumuhm, 9784 LHS, RHSSwap, Zero, DAG, dl, MVT::v4i32); 9785 // Shift the high parts up 16 bits. 9786 HiProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vslw, HiProd, 9787 Neg16, DAG, dl); 9788 return DAG.getNode(ISD::ADD, dl, MVT::v4i32, LoProd, HiProd); 9789 } else if (Op.getValueType() == MVT::v8i16) { 9790 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); 9791 9792 SDValue Zero = BuildSplatI(0, 1, MVT::v8i16, DAG, dl); 9793 9794 return BuildIntrinsicOp(Intrinsic::ppc_altivec_vmladduhm, 9795 LHS, RHS, Zero, DAG, dl); 9796 } else if (Op.getValueType() == MVT::v16i8) { 9797 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); 9798 bool isLittleEndian = Subtarget.isLittleEndian(); 9799 9800 // Multiply the even 8-bit parts, producing 16-bit sums. 9801 SDValue EvenParts = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmuleub, 9802 LHS, RHS, DAG, dl, MVT::v8i16); 9803 EvenParts = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, EvenParts); 9804 9805 // Multiply the odd 8-bit parts, producing 16-bit sums. 9806 SDValue OddParts = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmuloub, 9807 LHS, RHS, DAG, dl, MVT::v8i16); 9808 OddParts = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OddParts); 9809 9810 // Merge the results together. Because vmuleub and vmuloub are 9811 // instructions with a big-endian bias, we must reverse the 9812 // element numbering and reverse the meaning of "odd" and "even" 9813 // when generating little endian code. 9814 int Ops[16]; 9815 for (unsigned i = 0; i != 8; ++i) { 9816 if (isLittleEndian) { 9817 Ops[i*2 ] = 2*i; 9818 Ops[i*2+1] = 2*i+16; 9819 } else { 9820 Ops[i*2 ] = 2*i+1; 9821 Ops[i*2+1] = 2*i+1+16; 9822 } 9823 } 9824 if (isLittleEndian) 9825 return DAG.getVectorShuffle(MVT::v16i8, dl, OddParts, EvenParts, Ops); 9826 else 9827 return DAG.getVectorShuffle(MVT::v16i8, dl, EvenParts, OddParts, Ops); 9828 } else { 9829 llvm_unreachable("Unknown mul to lower!"); 9830 } 9831 } 9832 9833 SDValue PPCTargetLowering::LowerABS(SDValue Op, SelectionDAG &DAG) const { 9834 9835 assert(Op.getOpcode() == ISD::ABS && "Should only be called for ISD::ABS"); 9836 9837 EVT VT = Op.getValueType(); 9838 assert(VT.isVector() && 9839 "Only set vector abs as custom, scalar abs shouldn't reach here!"); 9840 assert((VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 || 9841 VT == MVT::v16i8) && 9842 "Unexpected vector element type!"); 9843 assert((VT != MVT::v2i64 || Subtarget.hasP8Altivec()) && 9844 "Current subtarget doesn't support smax v2i64!"); 9845 9846 // For vector abs, it can be lowered to: 9847 // abs x 9848 // ==> 9849 // y = -x 9850 // smax(x, y) 9851 9852 SDLoc dl(Op); 9853 SDValue X = Op.getOperand(0); 9854 SDValue Zero = DAG.getConstant(0, dl, VT); 9855 SDValue Y = DAG.getNode(ISD::SUB, dl, VT, Zero, X); 9856 9857 // SMAX patch https://reviews.llvm.org/D47332 9858 // hasn't landed yet, so use intrinsic first here. 9859 // TODO: Should use SMAX directly once SMAX patch landed 9860 Intrinsic::ID BifID = Intrinsic::ppc_altivec_vmaxsw; 9861 if (VT == MVT::v2i64) 9862 BifID = Intrinsic::ppc_altivec_vmaxsd; 9863 else if (VT == MVT::v8i16) 9864 BifID = Intrinsic::ppc_altivec_vmaxsh; 9865 else if (VT == MVT::v16i8) 9866 BifID = Intrinsic::ppc_altivec_vmaxsb; 9867 9868 return BuildIntrinsicOp(BifID, X, Y, DAG, dl, VT); 9869 } 9870 9871 // Custom lowering for fpext vf32 to v2f64 9872 SDValue PPCTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const { 9873 9874 assert(Op.getOpcode() == ISD::FP_EXTEND && 9875 "Should only be called for ISD::FP_EXTEND"); 9876 9877 // We only want to custom lower an extend from v2f32 to v2f64. 9878 if (Op.getValueType() != MVT::v2f64 || 9879 Op.getOperand(0).getValueType() != MVT::v2f32) 9880 return SDValue(); 9881 9882 SDLoc dl(Op); 9883 SDValue Op0 = Op.getOperand(0); 9884 9885 switch (Op0.getOpcode()) { 9886 default: 9887 return SDValue(); 9888 case ISD::FADD: 9889 case ISD::FMUL: 9890 case ISD::FSUB: { 9891 SDValue NewLoad[2]; 9892 for (unsigned i = 0, ie = Op0.getNumOperands(); i != ie; ++i) { 9893 // Ensure both input are loads. 9894 SDValue LdOp = Op0.getOperand(i); 9895 if (LdOp.getOpcode() != ISD::LOAD) 9896 return SDValue(); 9897 // Generate new load node. 9898 LoadSDNode *LD = cast<LoadSDNode>(LdOp); 9899 SDValue LoadOps[] = { LD->getChain(), LD->getBasePtr() }; 9900 NewLoad[i] = 9901 DAG.getMemIntrinsicNode(PPCISD::LD_VSX_LH, dl, 9902 DAG.getVTList(MVT::v4f32, MVT::Other), 9903 LoadOps, LD->getMemoryVT(), 9904 LD->getMemOperand()); 9905 } 9906 SDValue NewOp = DAG.getNode(Op0.getOpcode(), SDLoc(Op0), MVT::v4f32, 9907 NewLoad[0], NewLoad[1], 9908 Op0.getNode()->getFlags()); 9909 return DAG.getNode(PPCISD::FP_EXTEND_LH, dl, MVT::v2f64, NewOp); 9910 } 9911 case ISD::LOAD: { 9912 LoadSDNode *LD = cast<LoadSDNode>(Op0); 9913 SDValue LoadOps[] = { LD->getChain(), LD->getBasePtr() }; 9914 SDValue NewLd = 9915 DAG.getMemIntrinsicNode(PPCISD::LD_VSX_LH, dl, 9916 DAG.getVTList(MVT::v4f32, MVT::Other), 9917 LoadOps, LD->getMemoryVT(), LD->getMemOperand()); 9918 return DAG.getNode(PPCISD::FP_EXTEND_LH, dl, MVT::v2f64, NewLd); 9919 } 9920 } 9921 llvm_unreachable("ERROR:Should return for all cases within swtich."); 9922 } 9923 9924 /// LowerOperation - Provide custom lowering hooks for some operations. 9925 /// 9926 SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 9927 switch (Op.getOpcode()) { 9928 default: llvm_unreachable("Wasn't expecting to be able to lower this!"); 9929 case ISD::ConstantPool: return LowerConstantPool(Op, DAG); 9930 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG); 9931 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); 9932 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); 9933 case ISD::JumpTable: return LowerJumpTable(Op, DAG); 9934 case ISD::SETCC: return LowerSETCC(Op, DAG); 9935 case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG); 9936 case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG); 9937 9938 // Variable argument lowering. 9939 case ISD::VASTART: return LowerVASTART(Op, DAG); 9940 case ISD::VAARG: return LowerVAARG(Op, DAG); 9941 case ISD::VACOPY: return LowerVACOPY(Op, DAG); 9942 9943 case ISD::STACKRESTORE: return LowerSTACKRESTORE(Op, DAG); 9944 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); 9945 case ISD::GET_DYNAMIC_AREA_OFFSET: 9946 return LowerGET_DYNAMIC_AREA_OFFSET(Op, DAG); 9947 9948 // Exception handling lowering. 9949 case ISD::EH_DWARF_CFA: return LowerEH_DWARF_CFA(Op, DAG); 9950 case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG); 9951 case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG); 9952 9953 case ISD::LOAD: return LowerLOAD(Op, DAG); 9954 case ISD::STORE: return LowerSTORE(Op, DAG); 9955 case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG); 9956 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG); 9957 case ISD::FP_TO_UINT: 9958 case ISD::FP_TO_SINT: return LowerFP_TO_INT(Op, DAG, SDLoc(Op)); 9959 case ISD::UINT_TO_FP: 9960 case ISD::SINT_TO_FP: return LowerINT_TO_FP(Op, DAG); 9961 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); 9962 9963 // Lower 64-bit shifts. 9964 case ISD::SHL_PARTS: return LowerSHL_PARTS(Op, DAG); 9965 case ISD::SRL_PARTS: return LowerSRL_PARTS(Op, DAG); 9966 case ISD::SRA_PARTS: return LowerSRA_PARTS(Op, DAG); 9967 9968 // Vector-related lowering. 9969 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); 9970 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); 9971 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); 9972 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG); 9973 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); 9974 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); 9975 case ISD::MUL: return LowerMUL(Op, DAG); 9976 case ISD::ABS: return LowerABS(Op, DAG); 9977 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG); 9978 9979 // For counter-based loop handling. 9980 case ISD::INTRINSIC_W_CHAIN: return SDValue(); 9981 9982 case ISD::BITCAST: return LowerBITCAST(Op, DAG); 9983 9984 // Frame & Return address. 9985 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); 9986 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); 9987 9988 case ISD::INTRINSIC_VOID: 9989 return LowerINTRINSIC_VOID(Op, DAG); 9990 case ISD::SREM: 9991 case ISD::UREM: 9992 return LowerREM(Op, DAG); 9993 case ISD::BSWAP: 9994 return LowerBSWAP(Op, DAG); 9995 case ISD::ATOMIC_CMP_SWAP: 9996 return LowerATOMIC_CMP_SWAP(Op, DAG); 9997 } 9998 } 9999 10000 void PPCTargetLowering::ReplaceNodeResults(SDNode *N, 10001 SmallVectorImpl<SDValue>&Results, 10002 SelectionDAG &DAG) const { 10003 SDLoc dl(N); 10004 switch (N->getOpcode()) { 10005 default: 10006 llvm_unreachable("Do not know how to custom type legalize this operation!"); 10007 case ISD::READCYCLECOUNTER: { 10008 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other); 10009 SDValue RTB = DAG.getNode(PPCISD::READ_TIME_BASE, dl, VTs, N->getOperand(0)); 10010 10011 Results.push_back(RTB); 10012 Results.push_back(RTB.getValue(1)); 10013 Results.push_back(RTB.getValue(2)); 10014 break; 10015 } 10016 case ISD::INTRINSIC_W_CHAIN: { 10017 if (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() != 10018 Intrinsic::loop_decrement) 10019 break; 10020 10021 assert(N->getValueType(0) == MVT::i1 && 10022 "Unexpected result type for CTR decrement intrinsic"); 10023 EVT SVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), 10024 N->getValueType(0)); 10025 SDVTList VTs = DAG.getVTList(SVT, MVT::Other); 10026 SDValue NewInt = DAG.getNode(N->getOpcode(), dl, VTs, N->getOperand(0), 10027 N->getOperand(1)); 10028 10029 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewInt)); 10030 Results.push_back(NewInt.getValue(1)); 10031 break; 10032 } 10033 case ISD::VAARG: { 10034 if (!Subtarget.isSVR4ABI() || Subtarget.isPPC64()) 10035 return; 10036 10037 EVT VT = N->getValueType(0); 10038 10039 if (VT == MVT::i64) { 10040 SDValue NewNode = LowerVAARG(SDValue(N, 1), DAG); 10041 10042 Results.push_back(NewNode); 10043 Results.push_back(NewNode.getValue(1)); 10044 } 10045 return; 10046 } 10047 case ISD::FP_TO_SINT: 10048 case ISD::FP_TO_UINT: 10049 // LowerFP_TO_INT() can only handle f32 and f64. 10050 if (N->getOperand(0).getValueType() == MVT::ppcf128) 10051 return; 10052 Results.push_back(LowerFP_TO_INT(SDValue(N, 0), DAG, dl)); 10053 return; 10054 case ISD::TRUNCATE: { 10055 EVT TrgVT = N->getValueType(0); 10056 if (TrgVT.isVector() && 10057 isOperationCustom(N->getOpcode(), TrgVT) && 10058 N->getOperand(0).getValueType().getSizeInBits() <= 128) 10059 Results.push_back(LowerTRUNCATEVector(SDValue(N, 0), DAG)); 10060 return; 10061 } 10062 case ISD::BITCAST: 10063 // Don't handle bitcast here. 10064 return; 10065 } 10066 } 10067 10068 //===----------------------------------------------------------------------===// 10069 // Other Lowering Code 10070 //===----------------------------------------------------------------------===// 10071 10072 static Instruction* callIntrinsic(IRBuilder<> &Builder, Intrinsic::ID Id) { 10073 Module *M = Builder.GetInsertBlock()->getParent()->getParent(); 10074 Function *Func = Intrinsic::getDeclaration(M, Id); 10075 return Builder.CreateCall(Func, {}); 10076 } 10077 10078 // The mappings for emitLeading/TrailingFence is taken from 10079 // http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html 10080 Instruction *PPCTargetLowering::emitLeadingFence(IRBuilder<> &Builder, 10081 Instruction *Inst, 10082 AtomicOrdering Ord) const { 10083 if (Ord == AtomicOrdering::SequentiallyConsistent) 10084 return callIntrinsic(Builder, Intrinsic::ppc_sync); 10085 if (isReleaseOrStronger(Ord)) 10086 return callIntrinsic(Builder, Intrinsic::ppc_lwsync); 10087 return nullptr; 10088 } 10089 10090 Instruction *PPCTargetLowering::emitTrailingFence(IRBuilder<> &Builder, 10091 Instruction *Inst, 10092 AtomicOrdering Ord) const { 10093 if (Inst->hasAtomicLoad() && isAcquireOrStronger(Ord)) { 10094 // See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html and 10095 // http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2011.03.04a.html 10096 // and http://www.cl.cam.ac.uk/~pes20/cppppc/ for justification. 10097 if (isa<LoadInst>(Inst) && Subtarget.isPPC64()) 10098 return Builder.CreateCall( 10099 Intrinsic::getDeclaration( 10100 Builder.GetInsertBlock()->getParent()->getParent(), 10101 Intrinsic::ppc_cfence, {Inst->getType()}), 10102 {Inst}); 10103 // FIXME: Can use isync for rmw operation. 10104 return callIntrinsic(Builder, Intrinsic::ppc_lwsync); 10105 } 10106 return nullptr; 10107 } 10108 10109 MachineBasicBlock * 10110 PPCTargetLowering::EmitAtomicBinary(MachineInstr &MI, MachineBasicBlock *BB, 10111 unsigned AtomicSize, 10112 unsigned BinOpcode, 10113 unsigned CmpOpcode, 10114 unsigned CmpPred) const { 10115 // This also handles ATOMIC_SWAP, indicated by BinOpcode==0. 10116 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 10117 10118 auto LoadMnemonic = PPC::LDARX; 10119 auto StoreMnemonic = PPC::STDCX; 10120 switch (AtomicSize) { 10121 default: 10122 llvm_unreachable("Unexpected size of atomic entity"); 10123 case 1: 10124 LoadMnemonic = PPC::LBARX; 10125 StoreMnemonic = PPC::STBCX; 10126 assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4"); 10127 break; 10128 case 2: 10129 LoadMnemonic = PPC::LHARX; 10130 StoreMnemonic = PPC::STHCX; 10131 assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4"); 10132 break; 10133 case 4: 10134 LoadMnemonic = PPC::LWARX; 10135 StoreMnemonic = PPC::STWCX; 10136 break; 10137 case 8: 10138 LoadMnemonic = PPC::LDARX; 10139 StoreMnemonic = PPC::STDCX; 10140 break; 10141 } 10142 10143 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 10144 MachineFunction *F = BB->getParent(); 10145 MachineFunction::iterator It = ++BB->getIterator(); 10146 10147 Register dest = MI.getOperand(0).getReg(); 10148 Register ptrA = MI.getOperand(1).getReg(); 10149 Register ptrB = MI.getOperand(2).getReg(); 10150 Register incr = MI.getOperand(3).getReg(); 10151 DebugLoc dl = MI.getDebugLoc(); 10152 10153 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(LLVM_BB); 10154 MachineBasicBlock *loop2MBB = 10155 CmpOpcode ? F->CreateMachineBasicBlock(LLVM_BB) : nullptr; 10156 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); 10157 F->insert(It, loopMBB); 10158 if (CmpOpcode) 10159 F->insert(It, loop2MBB); 10160 F->insert(It, exitMBB); 10161 exitMBB->splice(exitMBB->begin(), BB, 10162 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 10163 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 10164 10165 MachineRegisterInfo &RegInfo = F->getRegInfo(); 10166 Register TmpReg = (!BinOpcode) ? incr : 10167 RegInfo.createVirtualRegister( AtomicSize == 8 ? &PPC::G8RCRegClass 10168 : &PPC::GPRCRegClass); 10169 10170 // thisMBB: 10171 // ... 10172 // fallthrough --> loopMBB 10173 BB->addSuccessor(loopMBB); 10174 10175 // loopMBB: 10176 // l[wd]arx dest, ptr 10177 // add r0, dest, incr 10178 // st[wd]cx. r0, ptr 10179 // bne- loopMBB 10180 // fallthrough --> exitMBB 10181 10182 // For max/min... 10183 // loopMBB: 10184 // l[wd]arx dest, ptr 10185 // cmpl?[wd] incr, dest 10186 // bgt exitMBB 10187 // loop2MBB: 10188 // st[wd]cx. dest, ptr 10189 // bne- loopMBB 10190 // fallthrough --> exitMBB 10191 10192 BB = loopMBB; 10193 BuildMI(BB, dl, TII->get(LoadMnemonic), dest) 10194 .addReg(ptrA).addReg(ptrB); 10195 if (BinOpcode) 10196 BuildMI(BB, dl, TII->get(BinOpcode), TmpReg).addReg(incr).addReg(dest); 10197 if (CmpOpcode) { 10198 // Signed comparisons of byte or halfword values must be sign-extended. 10199 if (CmpOpcode == PPC::CMPW && AtomicSize < 4) { 10200 unsigned ExtReg = RegInfo.createVirtualRegister(&PPC::GPRCRegClass); 10201 BuildMI(BB, dl, TII->get(AtomicSize == 1 ? PPC::EXTSB : PPC::EXTSH), 10202 ExtReg).addReg(dest); 10203 BuildMI(BB, dl, TII->get(CmpOpcode), PPC::CR0) 10204 .addReg(incr).addReg(ExtReg); 10205 } else 10206 BuildMI(BB, dl, TII->get(CmpOpcode), PPC::CR0) 10207 .addReg(incr).addReg(dest); 10208 10209 BuildMI(BB, dl, TII->get(PPC::BCC)) 10210 .addImm(CmpPred).addReg(PPC::CR0).addMBB(exitMBB); 10211 BB->addSuccessor(loop2MBB); 10212 BB->addSuccessor(exitMBB); 10213 BB = loop2MBB; 10214 } 10215 BuildMI(BB, dl, TII->get(StoreMnemonic)) 10216 .addReg(TmpReg).addReg(ptrA).addReg(ptrB); 10217 BuildMI(BB, dl, TII->get(PPC::BCC)) 10218 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(loopMBB); 10219 BB->addSuccessor(loopMBB); 10220 BB->addSuccessor(exitMBB); 10221 10222 // exitMBB: 10223 // ... 10224 BB = exitMBB; 10225 return BB; 10226 } 10227 10228 MachineBasicBlock *PPCTargetLowering::EmitPartwordAtomicBinary( 10229 MachineInstr &MI, MachineBasicBlock *BB, 10230 bool is8bit, // operation 10231 unsigned BinOpcode, unsigned CmpOpcode, unsigned CmpPred) const { 10232 // If we support part-word atomic mnemonics, just use them 10233 if (Subtarget.hasPartwordAtomics()) 10234 return EmitAtomicBinary(MI, BB, is8bit ? 1 : 2, BinOpcode, CmpOpcode, 10235 CmpPred); 10236 10237 // This also handles ATOMIC_SWAP, indicated by BinOpcode==0. 10238 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 10239 // In 64 bit mode we have to use 64 bits for addresses, even though the 10240 // lwarx/stwcx are 32 bits. With the 32-bit atomics we can use address 10241 // registers without caring whether they're 32 or 64, but here we're 10242 // doing actual arithmetic on the addresses. 10243 bool is64bit = Subtarget.isPPC64(); 10244 bool isLittleEndian = Subtarget.isLittleEndian(); 10245 unsigned ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO; 10246 10247 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 10248 MachineFunction *F = BB->getParent(); 10249 MachineFunction::iterator It = ++BB->getIterator(); 10250 10251 unsigned dest = MI.getOperand(0).getReg(); 10252 unsigned ptrA = MI.getOperand(1).getReg(); 10253 unsigned ptrB = MI.getOperand(2).getReg(); 10254 unsigned incr = MI.getOperand(3).getReg(); 10255 DebugLoc dl = MI.getDebugLoc(); 10256 10257 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(LLVM_BB); 10258 MachineBasicBlock *loop2MBB = 10259 CmpOpcode ? F->CreateMachineBasicBlock(LLVM_BB) : nullptr; 10260 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); 10261 F->insert(It, loopMBB); 10262 if (CmpOpcode) 10263 F->insert(It, loop2MBB); 10264 F->insert(It, exitMBB); 10265 exitMBB->splice(exitMBB->begin(), BB, 10266 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 10267 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 10268 10269 MachineRegisterInfo &RegInfo = F->getRegInfo(); 10270 const TargetRegisterClass *RC = 10271 is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; 10272 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass; 10273 10274 Register PtrReg = RegInfo.createVirtualRegister(RC); 10275 Register Shift1Reg = RegInfo.createVirtualRegister(GPRC); 10276 Register ShiftReg = 10277 isLittleEndian ? Shift1Reg : RegInfo.createVirtualRegister(GPRC); 10278 Register Incr2Reg = RegInfo.createVirtualRegister(GPRC); 10279 Register MaskReg = RegInfo.createVirtualRegister(GPRC); 10280 Register Mask2Reg = RegInfo.createVirtualRegister(GPRC); 10281 Register Mask3Reg = RegInfo.createVirtualRegister(GPRC); 10282 Register Tmp2Reg = RegInfo.createVirtualRegister(GPRC); 10283 Register Tmp3Reg = RegInfo.createVirtualRegister(GPRC); 10284 Register Tmp4Reg = RegInfo.createVirtualRegister(GPRC); 10285 Register TmpDestReg = RegInfo.createVirtualRegister(GPRC); 10286 Register Ptr1Reg; 10287 Register TmpReg = 10288 (!BinOpcode) ? Incr2Reg : RegInfo.createVirtualRegister(GPRC); 10289 10290 // thisMBB: 10291 // ... 10292 // fallthrough --> loopMBB 10293 BB->addSuccessor(loopMBB); 10294 10295 // The 4-byte load must be aligned, while a char or short may be 10296 // anywhere in the word. Hence all this nasty bookkeeping code. 10297 // add ptr1, ptrA, ptrB [copy if ptrA==0] 10298 // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27] 10299 // xori shift, shift1, 24 [16] 10300 // rlwinm ptr, ptr1, 0, 0, 29 10301 // slw incr2, incr, shift 10302 // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535] 10303 // slw mask, mask2, shift 10304 // loopMBB: 10305 // lwarx tmpDest, ptr 10306 // add tmp, tmpDest, incr2 10307 // andc tmp2, tmpDest, mask 10308 // and tmp3, tmp, mask 10309 // or tmp4, tmp3, tmp2 10310 // stwcx. tmp4, ptr 10311 // bne- loopMBB 10312 // fallthrough --> exitMBB 10313 // srw dest, tmpDest, shift 10314 if (ptrA != ZeroReg) { 10315 Ptr1Reg = RegInfo.createVirtualRegister(RC); 10316 BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg) 10317 .addReg(ptrA) 10318 .addReg(ptrB); 10319 } else { 10320 Ptr1Reg = ptrB; 10321 } 10322 // We need use 32-bit subregister to avoid mismatch register class in 64-bit 10323 // mode. 10324 BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg) 10325 .addReg(Ptr1Reg, 0, is64bit ? PPC::sub_32 : 0) 10326 .addImm(3) 10327 .addImm(27) 10328 .addImm(is8bit ? 28 : 27); 10329 if (!isLittleEndian) 10330 BuildMI(BB, dl, TII->get(PPC::XORI), ShiftReg) 10331 .addReg(Shift1Reg) 10332 .addImm(is8bit ? 24 : 16); 10333 if (is64bit) 10334 BuildMI(BB, dl, TII->get(PPC::RLDICR), PtrReg) 10335 .addReg(Ptr1Reg) 10336 .addImm(0) 10337 .addImm(61); 10338 else 10339 BuildMI(BB, dl, TII->get(PPC::RLWINM), PtrReg) 10340 .addReg(Ptr1Reg) 10341 .addImm(0) 10342 .addImm(0) 10343 .addImm(29); 10344 BuildMI(BB, dl, TII->get(PPC::SLW), Incr2Reg).addReg(incr).addReg(ShiftReg); 10345 if (is8bit) 10346 BuildMI(BB, dl, TII->get(PPC::LI), Mask2Reg).addImm(255); 10347 else { 10348 BuildMI(BB, dl, TII->get(PPC::LI), Mask3Reg).addImm(0); 10349 BuildMI(BB, dl, TII->get(PPC::ORI), Mask2Reg) 10350 .addReg(Mask3Reg) 10351 .addImm(65535); 10352 } 10353 BuildMI(BB, dl, TII->get(PPC::SLW), MaskReg) 10354 .addReg(Mask2Reg) 10355 .addReg(ShiftReg); 10356 10357 BB = loopMBB; 10358 BuildMI(BB, dl, TII->get(PPC::LWARX), TmpDestReg) 10359 .addReg(ZeroReg) 10360 .addReg(PtrReg); 10361 if (BinOpcode) 10362 BuildMI(BB, dl, TII->get(BinOpcode), TmpReg) 10363 .addReg(Incr2Reg) 10364 .addReg(TmpDestReg); 10365 BuildMI(BB, dl, TII->get(PPC::ANDC), Tmp2Reg) 10366 .addReg(TmpDestReg) 10367 .addReg(MaskReg); 10368 BuildMI(BB, dl, TII->get(PPC::AND), Tmp3Reg).addReg(TmpReg).addReg(MaskReg); 10369 if (CmpOpcode) { 10370 // For unsigned comparisons, we can directly compare the shifted values. 10371 // For signed comparisons we shift and sign extend. 10372 unsigned SReg = RegInfo.createVirtualRegister(GPRC); 10373 BuildMI(BB, dl, TII->get(PPC::AND), SReg) 10374 .addReg(TmpDestReg) 10375 .addReg(MaskReg); 10376 unsigned ValueReg = SReg; 10377 unsigned CmpReg = Incr2Reg; 10378 if (CmpOpcode == PPC::CMPW) { 10379 ValueReg = RegInfo.createVirtualRegister(GPRC); 10380 BuildMI(BB, dl, TII->get(PPC::SRW), ValueReg) 10381 .addReg(SReg) 10382 .addReg(ShiftReg); 10383 unsigned ValueSReg = RegInfo.createVirtualRegister(GPRC); 10384 BuildMI(BB, dl, TII->get(is8bit ? PPC::EXTSB : PPC::EXTSH), ValueSReg) 10385 .addReg(ValueReg); 10386 ValueReg = ValueSReg; 10387 CmpReg = incr; 10388 } 10389 BuildMI(BB, dl, TII->get(CmpOpcode), PPC::CR0) 10390 .addReg(CmpReg) 10391 .addReg(ValueReg); 10392 BuildMI(BB, dl, TII->get(PPC::BCC)) 10393 .addImm(CmpPred) 10394 .addReg(PPC::CR0) 10395 .addMBB(exitMBB); 10396 BB->addSuccessor(loop2MBB); 10397 BB->addSuccessor(exitMBB); 10398 BB = loop2MBB; 10399 } 10400 BuildMI(BB, dl, TII->get(PPC::OR), Tmp4Reg).addReg(Tmp3Reg).addReg(Tmp2Reg); 10401 BuildMI(BB, dl, TII->get(PPC::STWCX)) 10402 .addReg(Tmp4Reg) 10403 .addReg(ZeroReg) 10404 .addReg(PtrReg); 10405 BuildMI(BB, dl, TII->get(PPC::BCC)) 10406 .addImm(PPC::PRED_NE) 10407 .addReg(PPC::CR0) 10408 .addMBB(loopMBB); 10409 BB->addSuccessor(loopMBB); 10410 BB->addSuccessor(exitMBB); 10411 10412 // exitMBB: 10413 // ... 10414 BB = exitMBB; 10415 BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), dest) 10416 .addReg(TmpDestReg) 10417 .addReg(ShiftReg); 10418 return BB; 10419 } 10420 10421 llvm::MachineBasicBlock * 10422 PPCTargetLowering::emitEHSjLjSetJmp(MachineInstr &MI, 10423 MachineBasicBlock *MBB) const { 10424 DebugLoc DL = MI.getDebugLoc(); 10425 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 10426 const PPCRegisterInfo *TRI = Subtarget.getRegisterInfo(); 10427 10428 MachineFunction *MF = MBB->getParent(); 10429 MachineRegisterInfo &MRI = MF->getRegInfo(); 10430 10431 const BasicBlock *BB = MBB->getBasicBlock(); 10432 MachineFunction::iterator I = ++MBB->getIterator(); 10433 10434 unsigned DstReg = MI.getOperand(0).getReg(); 10435 const TargetRegisterClass *RC = MRI.getRegClass(DstReg); 10436 assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!"); 10437 unsigned mainDstReg = MRI.createVirtualRegister(RC); 10438 unsigned restoreDstReg = MRI.createVirtualRegister(RC); 10439 10440 MVT PVT = getPointerTy(MF->getDataLayout()); 10441 assert((PVT == MVT::i64 || PVT == MVT::i32) && 10442 "Invalid Pointer Size!"); 10443 // For v = setjmp(buf), we generate 10444 // 10445 // thisMBB: 10446 // SjLjSetup mainMBB 10447 // bl mainMBB 10448 // v_restore = 1 10449 // b sinkMBB 10450 // 10451 // mainMBB: 10452 // buf[LabelOffset] = LR 10453 // v_main = 0 10454 // 10455 // sinkMBB: 10456 // v = phi(main, restore) 10457 // 10458 10459 MachineBasicBlock *thisMBB = MBB; 10460 MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB); 10461 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB); 10462 MF->insert(I, mainMBB); 10463 MF->insert(I, sinkMBB); 10464 10465 MachineInstrBuilder MIB; 10466 10467 // Transfer the remainder of BB and its successor edges to sinkMBB. 10468 sinkMBB->splice(sinkMBB->begin(), MBB, 10469 std::next(MachineBasicBlock::iterator(MI)), MBB->end()); 10470 sinkMBB->transferSuccessorsAndUpdatePHIs(MBB); 10471 10472 // Note that the structure of the jmp_buf used here is not compatible 10473 // with that used by libc, and is not designed to be. Specifically, it 10474 // stores only those 'reserved' registers that LLVM does not otherwise 10475 // understand how to spill. Also, by convention, by the time this 10476 // intrinsic is called, Clang has already stored the frame address in the 10477 // first slot of the buffer and stack address in the third. Following the 10478 // X86 target code, we'll store the jump address in the second slot. We also 10479 // need to save the TOC pointer (R2) to handle jumps between shared 10480 // libraries, and that will be stored in the fourth slot. The thread 10481 // identifier (R13) is not affected. 10482 10483 // thisMBB: 10484 const int64_t LabelOffset = 1 * PVT.getStoreSize(); 10485 const int64_t TOCOffset = 3 * PVT.getStoreSize(); 10486 const int64_t BPOffset = 4 * PVT.getStoreSize(); 10487 10488 // Prepare IP either in reg. 10489 const TargetRegisterClass *PtrRC = getRegClassFor(PVT); 10490 unsigned LabelReg = MRI.createVirtualRegister(PtrRC); 10491 unsigned BufReg = MI.getOperand(1).getReg(); 10492 10493 if (Subtarget.isPPC64() && Subtarget.isSVR4ABI()) { 10494 setUsesTOCBasePtr(*MBB->getParent()); 10495 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::STD)) 10496 .addReg(PPC::X2) 10497 .addImm(TOCOffset) 10498 .addReg(BufReg) 10499 .cloneMemRefs(MI); 10500 } 10501 10502 // Naked functions never have a base pointer, and so we use r1. For all 10503 // other functions, this decision must be delayed until during PEI. 10504 unsigned BaseReg; 10505 if (MF->getFunction().hasFnAttribute(Attribute::Naked)) 10506 BaseReg = Subtarget.isPPC64() ? PPC::X1 : PPC::R1; 10507 else 10508 BaseReg = Subtarget.isPPC64() ? PPC::BP8 : PPC::BP; 10509 10510 MIB = BuildMI(*thisMBB, MI, DL, 10511 TII->get(Subtarget.isPPC64() ? PPC::STD : PPC::STW)) 10512 .addReg(BaseReg) 10513 .addImm(BPOffset) 10514 .addReg(BufReg) 10515 .cloneMemRefs(MI); 10516 10517 // Setup 10518 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::BCLalways)).addMBB(mainMBB); 10519 MIB.addRegMask(TRI->getNoPreservedMask()); 10520 10521 BuildMI(*thisMBB, MI, DL, TII->get(PPC::LI), restoreDstReg).addImm(1); 10522 10523 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::EH_SjLj_Setup)) 10524 .addMBB(mainMBB); 10525 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::B)).addMBB(sinkMBB); 10526 10527 thisMBB->addSuccessor(mainMBB, BranchProbability::getZero()); 10528 thisMBB->addSuccessor(sinkMBB, BranchProbability::getOne()); 10529 10530 // mainMBB: 10531 // mainDstReg = 0 10532 MIB = 10533 BuildMI(mainMBB, DL, 10534 TII->get(Subtarget.isPPC64() ? PPC::MFLR8 : PPC::MFLR), LabelReg); 10535 10536 // Store IP 10537 if (Subtarget.isPPC64()) { 10538 MIB = BuildMI(mainMBB, DL, TII->get(PPC::STD)) 10539 .addReg(LabelReg) 10540 .addImm(LabelOffset) 10541 .addReg(BufReg); 10542 } else { 10543 MIB = BuildMI(mainMBB, DL, TII->get(PPC::STW)) 10544 .addReg(LabelReg) 10545 .addImm(LabelOffset) 10546 .addReg(BufReg); 10547 } 10548 MIB.cloneMemRefs(MI); 10549 10550 BuildMI(mainMBB, DL, TII->get(PPC::LI), mainDstReg).addImm(0); 10551 mainMBB->addSuccessor(sinkMBB); 10552 10553 // sinkMBB: 10554 BuildMI(*sinkMBB, sinkMBB->begin(), DL, 10555 TII->get(PPC::PHI), DstReg) 10556 .addReg(mainDstReg).addMBB(mainMBB) 10557 .addReg(restoreDstReg).addMBB(thisMBB); 10558 10559 MI.eraseFromParent(); 10560 return sinkMBB; 10561 } 10562 10563 MachineBasicBlock * 10564 PPCTargetLowering::emitEHSjLjLongJmp(MachineInstr &MI, 10565 MachineBasicBlock *MBB) const { 10566 DebugLoc DL = MI.getDebugLoc(); 10567 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 10568 10569 MachineFunction *MF = MBB->getParent(); 10570 MachineRegisterInfo &MRI = MF->getRegInfo(); 10571 10572 MVT PVT = getPointerTy(MF->getDataLayout()); 10573 assert((PVT == MVT::i64 || PVT == MVT::i32) && 10574 "Invalid Pointer Size!"); 10575 10576 const TargetRegisterClass *RC = 10577 (PVT == MVT::i64) ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; 10578 unsigned Tmp = MRI.createVirtualRegister(RC); 10579 // Since FP is only updated here but NOT referenced, it's treated as GPR. 10580 unsigned FP = (PVT == MVT::i64) ? PPC::X31 : PPC::R31; 10581 unsigned SP = (PVT == MVT::i64) ? PPC::X1 : PPC::R1; 10582 unsigned BP = 10583 (PVT == MVT::i64) 10584 ? PPC::X30 10585 : (Subtarget.isSVR4ABI() && isPositionIndependent() ? PPC::R29 10586 : PPC::R30); 10587 10588 MachineInstrBuilder MIB; 10589 10590 const int64_t LabelOffset = 1 * PVT.getStoreSize(); 10591 const int64_t SPOffset = 2 * PVT.getStoreSize(); 10592 const int64_t TOCOffset = 3 * PVT.getStoreSize(); 10593 const int64_t BPOffset = 4 * PVT.getStoreSize(); 10594 10595 unsigned BufReg = MI.getOperand(0).getReg(); 10596 10597 // Reload FP (the jumped-to function may not have had a 10598 // frame pointer, and if so, then its r31 will be restored 10599 // as necessary). 10600 if (PVT == MVT::i64) { 10601 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), FP) 10602 .addImm(0) 10603 .addReg(BufReg); 10604 } else { 10605 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), FP) 10606 .addImm(0) 10607 .addReg(BufReg); 10608 } 10609 MIB.cloneMemRefs(MI); 10610 10611 // Reload IP 10612 if (PVT == MVT::i64) { 10613 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), Tmp) 10614 .addImm(LabelOffset) 10615 .addReg(BufReg); 10616 } else { 10617 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), Tmp) 10618 .addImm(LabelOffset) 10619 .addReg(BufReg); 10620 } 10621 MIB.cloneMemRefs(MI); 10622 10623 // Reload SP 10624 if (PVT == MVT::i64) { 10625 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), SP) 10626 .addImm(SPOffset) 10627 .addReg(BufReg); 10628 } else { 10629 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), SP) 10630 .addImm(SPOffset) 10631 .addReg(BufReg); 10632 } 10633 MIB.cloneMemRefs(MI); 10634 10635 // Reload BP 10636 if (PVT == MVT::i64) { 10637 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), BP) 10638 .addImm(BPOffset) 10639 .addReg(BufReg); 10640 } else { 10641 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), BP) 10642 .addImm(BPOffset) 10643 .addReg(BufReg); 10644 } 10645 MIB.cloneMemRefs(MI); 10646 10647 // Reload TOC 10648 if (PVT == MVT::i64 && Subtarget.isSVR4ABI()) { 10649 setUsesTOCBasePtr(*MBB->getParent()); 10650 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), PPC::X2) 10651 .addImm(TOCOffset) 10652 .addReg(BufReg) 10653 .cloneMemRefs(MI); 10654 } 10655 10656 // Jump 10657 BuildMI(*MBB, MI, DL, 10658 TII->get(PVT == MVT::i64 ? PPC::MTCTR8 : PPC::MTCTR)).addReg(Tmp); 10659 BuildMI(*MBB, MI, DL, TII->get(PVT == MVT::i64 ? PPC::BCTR8 : PPC::BCTR)); 10660 10661 MI.eraseFromParent(); 10662 return MBB; 10663 } 10664 10665 MachineBasicBlock * 10666 PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, 10667 MachineBasicBlock *BB) const { 10668 if (MI.getOpcode() == TargetOpcode::STACKMAP || 10669 MI.getOpcode() == TargetOpcode::PATCHPOINT) { 10670 if (Subtarget.isPPC64() && Subtarget.isSVR4ABI() && 10671 MI.getOpcode() == TargetOpcode::PATCHPOINT) { 10672 // Call lowering should have added an r2 operand to indicate a dependence 10673 // on the TOC base pointer value. It can't however, because there is no 10674 // way to mark the dependence as implicit there, and so the stackmap code 10675 // will confuse it with a regular operand. Instead, add the dependence 10676 // here. 10677 MI.addOperand(MachineOperand::CreateReg(PPC::X2, false, true)); 10678 } 10679 10680 return emitPatchPoint(MI, BB); 10681 } 10682 10683 if (MI.getOpcode() == PPC::EH_SjLj_SetJmp32 || 10684 MI.getOpcode() == PPC::EH_SjLj_SetJmp64) { 10685 return emitEHSjLjSetJmp(MI, BB); 10686 } else if (MI.getOpcode() == PPC::EH_SjLj_LongJmp32 || 10687 MI.getOpcode() == PPC::EH_SjLj_LongJmp64) { 10688 return emitEHSjLjLongJmp(MI, BB); 10689 } 10690 10691 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 10692 10693 // To "insert" these instructions we actually have to insert their 10694 // control-flow patterns. 10695 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 10696 MachineFunction::iterator It = ++BB->getIterator(); 10697 10698 MachineFunction *F = BB->getParent(); 10699 10700 if (MI.getOpcode() == PPC::SELECT_CC_I4 || 10701 MI.getOpcode() == PPC::SELECT_CC_I8 || MI.getOpcode() == PPC::SELECT_I4 || 10702 MI.getOpcode() == PPC::SELECT_I8) { 10703 SmallVector<MachineOperand, 2> Cond; 10704 if (MI.getOpcode() == PPC::SELECT_CC_I4 || 10705 MI.getOpcode() == PPC::SELECT_CC_I8) 10706 Cond.push_back(MI.getOperand(4)); 10707 else 10708 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET)); 10709 Cond.push_back(MI.getOperand(1)); 10710 10711 DebugLoc dl = MI.getDebugLoc(); 10712 TII->insertSelect(*BB, MI, dl, MI.getOperand(0).getReg(), Cond, 10713 MI.getOperand(2).getReg(), MI.getOperand(3).getReg()); 10714 } else if (MI.getOpcode() == PPC::SELECT_CC_I4 || 10715 MI.getOpcode() == PPC::SELECT_CC_I8 || 10716 MI.getOpcode() == PPC::SELECT_CC_F4 || 10717 MI.getOpcode() == PPC::SELECT_CC_F8 || 10718 MI.getOpcode() == PPC::SELECT_CC_F16 || 10719 MI.getOpcode() == PPC::SELECT_CC_QFRC || 10720 MI.getOpcode() == PPC::SELECT_CC_QSRC || 10721 MI.getOpcode() == PPC::SELECT_CC_QBRC || 10722 MI.getOpcode() == PPC::SELECT_CC_VRRC || 10723 MI.getOpcode() == PPC::SELECT_CC_VSFRC || 10724 MI.getOpcode() == PPC::SELECT_CC_VSSRC || 10725 MI.getOpcode() == PPC::SELECT_CC_VSRC || 10726 MI.getOpcode() == PPC::SELECT_CC_SPE4 || 10727 MI.getOpcode() == PPC::SELECT_CC_SPE || 10728 MI.getOpcode() == PPC::SELECT_I4 || 10729 MI.getOpcode() == PPC::SELECT_I8 || 10730 MI.getOpcode() == PPC::SELECT_F4 || 10731 MI.getOpcode() == PPC::SELECT_F8 || 10732 MI.getOpcode() == PPC::SELECT_F16 || 10733 MI.getOpcode() == PPC::SELECT_QFRC || 10734 MI.getOpcode() == PPC::SELECT_QSRC || 10735 MI.getOpcode() == PPC::SELECT_QBRC || 10736 MI.getOpcode() == PPC::SELECT_SPE || 10737 MI.getOpcode() == PPC::SELECT_SPE4 || 10738 MI.getOpcode() == PPC::SELECT_VRRC || 10739 MI.getOpcode() == PPC::SELECT_VSFRC || 10740 MI.getOpcode() == PPC::SELECT_VSSRC || 10741 MI.getOpcode() == PPC::SELECT_VSRC) { 10742 // The incoming instruction knows the destination vreg to set, the 10743 // condition code register to branch on, the true/false values to 10744 // select between, and a branch opcode to use. 10745 10746 // thisMBB: 10747 // ... 10748 // TrueVal = ... 10749 // cmpTY ccX, r1, r2 10750 // bCC copy1MBB 10751 // fallthrough --> copy0MBB 10752 MachineBasicBlock *thisMBB = BB; 10753 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); 10754 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); 10755 DebugLoc dl = MI.getDebugLoc(); 10756 F->insert(It, copy0MBB); 10757 F->insert(It, sinkMBB); 10758 10759 // Transfer the remainder of BB and its successor edges to sinkMBB. 10760 sinkMBB->splice(sinkMBB->begin(), BB, 10761 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 10762 sinkMBB->transferSuccessorsAndUpdatePHIs(BB); 10763 10764 // Next, add the true and fallthrough blocks as its successors. 10765 BB->addSuccessor(copy0MBB); 10766 BB->addSuccessor(sinkMBB); 10767 10768 if (MI.getOpcode() == PPC::SELECT_I4 || MI.getOpcode() == PPC::SELECT_I8 || 10769 MI.getOpcode() == PPC::SELECT_F4 || MI.getOpcode() == PPC::SELECT_F8 || 10770 MI.getOpcode() == PPC::SELECT_F16 || 10771 MI.getOpcode() == PPC::SELECT_SPE4 || 10772 MI.getOpcode() == PPC::SELECT_SPE || 10773 MI.getOpcode() == PPC::SELECT_QFRC || 10774 MI.getOpcode() == PPC::SELECT_QSRC || 10775 MI.getOpcode() == PPC::SELECT_QBRC || 10776 MI.getOpcode() == PPC::SELECT_VRRC || 10777 MI.getOpcode() == PPC::SELECT_VSFRC || 10778 MI.getOpcode() == PPC::SELECT_VSSRC || 10779 MI.getOpcode() == PPC::SELECT_VSRC) { 10780 BuildMI(BB, dl, TII->get(PPC::BC)) 10781 .addReg(MI.getOperand(1).getReg()) 10782 .addMBB(sinkMBB); 10783 } else { 10784 unsigned SelectPred = MI.getOperand(4).getImm(); 10785 BuildMI(BB, dl, TII->get(PPC::BCC)) 10786 .addImm(SelectPred) 10787 .addReg(MI.getOperand(1).getReg()) 10788 .addMBB(sinkMBB); 10789 } 10790 10791 // copy0MBB: 10792 // %FalseValue = ... 10793 // # fallthrough to sinkMBB 10794 BB = copy0MBB; 10795 10796 // Update machine-CFG edges 10797 BB->addSuccessor(sinkMBB); 10798 10799 // sinkMBB: 10800 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] 10801 // ... 10802 BB = sinkMBB; 10803 BuildMI(*BB, BB->begin(), dl, TII->get(PPC::PHI), MI.getOperand(0).getReg()) 10804 .addReg(MI.getOperand(3).getReg()) 10805 .addMBB(copy0MBB) 10806 .addReg(MI.getOperand(2).getReg()) 10807 .addMBB(thisMBB); 10808 } else if (MI.getOpcode() == PPC::ReadTB) { 10809 // To read the 64-bit time-base register on a 32-bit target, we read the 10810 // two halves. Should the counter have wrapped while it was being read, we 10811 // need to try again. 10812 // ... 10813 // readLoop: 10814 // mfspr Rx,TBU # load from TBU 10815 // mfspr Ry,TB # load from TB 10816 // mfspr Rz,TBU # load from TBU 10817 // cmpw crX,Rx,Rz # check if 'old'='new' 10818 // bne readLoop # branch if they're not equal 10819 // ... 10820 10821 MachineBasicBlock *readMBB = F->CreateMachineBasicBlock(LLVM_BB); 10822 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); 10823 DebugLoc dl = MI.getDebugLoc(); 10824 F->insert(It, readMBB); 10825 F->insert(It, sinkMBB); 10826 10827 // Transfer the remainder of BB and its successor edges to sinkMBB. 10828 sinkMBB->splice(sinkMBB->begin(), BB, 10829 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 10830 sinkMBB->transferSuccessorsAndUpdatePHIs(BB); 10831 10832 BB->addSuccessor(readMBB); 10833 BB = readMBB; 10834 10835 MachineRegisterInfo &RegInfo = F->getRegInfo(); 10836 unsigned ReadAgainReg = RegInfo.createVirtualRegister(&PPC::GPRCRegClass); 10837 unsigned LoReg = MI.getOperand(0).getReg(); 10838 unsigned HiReg = MI.getOperand(1).getReg(); 10839 10840 BuildMI(BB, dl, TII->get(PPC::MFSPR), HiReg).addImm(269); 10841 BuildMI(BB, dl, TII->get(PPC::MFSPR), LoReg).addImm(268); 10842 BuildMI(BB, dl, TII->get(PPC::MFSPR), ReadAgainReg).addImm(269); 10843 10844 unsigned CmpReg = RegInfo.createVirtualRegister(&PPC::CRRCRegClass); 10845 10846 BuildMI(BB, dl, TII->get(PPC::CMPW), CmpReg) 10847 .addReg(HiReg) 10848 .addReg(ReadAgainReg); 10849 BuildMI(BB, dl, TII->get(PPC::BCC)) 10850 .addImm(PPC::PRED_NE) 10851 .addReg(CmpReg) 10852 .addMBB(readMBB); 10853 10854 BB->addSuccessor(readMBB); 10855 BB->addSuccessor(sinkMBB); 10856 } else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I8) 10857 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::ADD4); 10858 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I16) 10859 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::ADD4); 10860 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I32) 10861 BB = EmitAtomicBinary(MI, BB, 4, PPC::ADD4); 10862 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I64) 10863 BB = EmitAtomicBinary(MI, BB, 8, PPC::ADD8); 10864 10865 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I8) 10866 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::AND); 10867 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I16) 10868 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::AND); 10869 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I32) 10870 BB = EmitAtomicBinary(MI, BB, 4, PPC::AND); 10871 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I64) 10872 BB = EmitAtomicBinary(MI, BB, 8, PPC::AND8); 10873 10874 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I8) 10875 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::OR); 10876 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I16) 10877 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::OR); 10878 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I32) 10879 BB = EmitAtomicBinary(MI, BB, 4, PPC::OR); 10880 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I64) 10881 BB = EmitAtomicBinary(MI, BB, 8, PPC::OR8); 10882 10883 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I8) 10884 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::XOR); 10885 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I16) 10886 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::XOR); 10887 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I32) 10888 BB = EmitAtomicBinary(MI, BB, 4, PPC::XOR); 10889 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I64) 10890 BB = EmitAtomicBinary(MI, BB, 8, PPC::XOR8); 10891 10892 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I8) 10893 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::NAND); 10894 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I16) 10895 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::NAND); 10896 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I32) 10897 BB = EmitAtomicBinary(MI, BB, 4, PPC::NAND); 10898 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I64) 10899 BB = EmitAtomicBinary(MI, BB, 8, PPC::NAND8); 10900 10901 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I8) 10902 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::SUBF); 10903 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I16) 10904 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::SUBF); 10905 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I32) 10906 BB = EmitAtomicBinary(MI, BB, 4, PPC::SUBF); 10907 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I64) 10908 BB = EmitAtomicBinary(MI, BB, 8, PPC::SUBF8); 10909 10910 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I8) 10911 BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPW, PPC::PRED_GE); 10912 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I16) 10913 BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPW, PPC::PRED_GE); 10914 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I32) 10915 BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPW, PPC::PRED_GE); 10916 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I64) 10917 BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPD, PPC::PRED_GE); 10918 10919 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I8) 10920 BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPW, PPC::PRED_LE); 10921 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I16) 10922 BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPW, PPC::PRED_LE); 10923 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I32) 10924 BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPW, PPC::PRED_LE); 10925 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I64) 10926 BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPD, PPC::PRED_LE); 10927 10928 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I8) 10929 BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPLW, PPC::PRED_GE); 10930 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I16) 10931 BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPLW, PPC::PRED_GE); 10932 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I32) 10933 BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPLW, PPC::PRED_GE); 10934 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I64) 10935 BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPLD, PPC::PRED_GE); 10936 10937 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I8) 10938 BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPLW, PPC::PRED_LE); 10939 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I16) 10940 BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPLW, PPC::PRED_LE); 10941 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I32) 10942 BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPLW, PPC::PRED_LE); 10943 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I64) 10944 BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPLD, PPC::PRED_LE); 10945 10946 else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I8) 10947 BB = EmitPartwordAtomicBinary(MI, BB, true, 0); 10948 else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I16) 10949 BB = EmitPartwordAtomicBinary(MI, BB, false, 0); 10950 else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I32) 10951 BB = EmitAtomicBinary(MI, BB, 4, 0); 10952 else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I64) 10953 BB = EmitAtomicBinary(MI, BB, 8, 0); 10954 else if (MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I32 || 10955 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I64 || 10956 (Subtarget.hasPartwordAtomics() && 10957 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8) || 10958 (Subtarget.hasPartwordAtomics() && 10959 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I16)) { 10960 bool is64bit = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I64; 10961 10962 auto LoadMnemonic = PPC::LDARX; 10963 auto StoreMnemonic = PPC::STDCX; 10964 switch (MI.getOpcode()) { 10965 default: 10966 llvm_unreachable("Compare and swap of unknown size"); 10967 case PPC::ATOMIC_CMP_SWAP_I8: 10968 LoadMnemonic = PPC::LBARX; 10969 StoreMnemonic = PPC::STBCX; 10970 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics."); 10971 break; 10972 case PPC::ATOMIC_CMP_SWAP_I16: 10973 LoadMnemonic = PPC::LHARX; 10974 StoreMnemonic = PPC::STHCX; 10975 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics."); 10976 break; 10977 case PPC::ATOMIC_CMP_SWAP_I32: 10978 LoadMnemonic = PPC::LWARX; 10979 StoreMnemonic = PPC::STWCX; 10980 break; 10981 case PPC::ATOMIC_CMP_SWAP_I64: 10982 LoadMnemonic = PPC::LDARX; 10983 StoreMnemonic = PPC::STDCX; 10984 break; 10985 } 10986 unsigned dest = MI.getOperand(0).getReg(); 10987 unsigned ptrA = MI.getOperand(1).getReg(); 10988 unsigned ptrB = MI.getOperand(2).getReg(); 10989 unsigned oldval = MI.getOperand(3).getReg(); 10990 unsigned newval = MI.getOperand(4).getReg(); 10991 DebugLoc dl = MI.getDebugLoc(); 10992 10993 MachineBasicBlock *loop1MBB = F->CreateMachineBasicBlock(LLVM_BB); 10994 MachineBasicBlock *loop2MBB = F->CreateMachineBasicBlock(LLVM_BB); 10995 MachineBasicBlock *midMBB = F->CreateMachineBasicBlock(LLVM_BB); 10996 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); 10997 F->insert(It, loop1MBB); 10998 F->insert(It, loop2MBB); 10999 F->insert(It, midMBB); 11000 F->insert(It, exitMBB); 11001 exitMBB->splice(exitMBB->begin(), BB, 11002 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 11003 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 11004 11005 // thisMBB: 11006 // ... 11007 // fallthrough --> loopMBB 11008 BB->addSuccessor(loop1MBB); 11009 11010 // loop1MBB: 11011 // l[bhwd]arx dest, ptr 11012 // cmp[wd] dest, oldval 11013 // bne- midMBB 11014 // loop2MBB: 11015 // st[bhwd]cx. newval, ptr 11016 // bne- loopMBB 11017 // b exitBB 11018 // midMBB: 11019 // st[bhwd]cx. dest, ptr 11020 // exitBB: 11021 BB = loop1MBB; 11022 BuildMI(BB, dl, TII->get(LoadMnemonic), dest).addReg(ptrA).addReg(ptrB); 11023 BuildMI(BB, dl, TII->get(is64bit ? PPC::CMPD : PPC::CMPW), PPC::CR0) 11024 .addReg(oldval) 11025 .addReg(dest); 11026 BuildMI(BB, dl, TII->get(PPC::BCC)) 11027 .addImm(PPC::PRED_NE) 11028 .addReg(PPC::CR0) 11029 .addMBB(midMBB); 11030 BB->addSuccessor(loop2MBB); 11031 BB->addSuccessor(midMBB); 11032 11033 BB = loop2MBB; 11034 BuildMI(BB, dl, TII->get(StoreMnemonic)) 11035 .addReg(newval) 11036 .addReg(ptrA) 11037 .addReg(ptrB); 11038 BuildMI(BB, dl, TII->get(PPC::BCC)) 11039 .addImm(PPC::PRED_NE) 11040 .addReg(PPC::CR0) 11041 .addMBB(loop1MBB); 11042 BuildMI(BB, dl, TII->get(PPC::B)).addMBB(exitMBB); 11043 BB->addSuccessor(loop1MBB); 11044 BB->addSuccessor(exitMBB); 11045 11046 BB = midMBB; 11047 BuildMI(BB, dl, TII->get(StoreMnemonic)) 11048 .addReg(dest) 11049 .addReg(ptrA) 11050 .addReg(ptrB); 11051 BB->addSuccessor(exitMBB); 11052 11053 // exitMBB: 11054 // ... 11055 BB = exitMBB; 11056 } else if (MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8 || 11057 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I16) { 11058 // We must use 64-bit registers for addresses when targeting 64-bit, 11059 // since we're actually doing arithmetic on them. Other registers 11060 // can be 32-bit. 11061 bool is64bit = Subtarget.isPPC64(); 11062 bool isLittleEndian = Subtarget.isLittleEndian(); 11063 bool is8bit = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8; 11064 11065 unsigned dest = MI.getOperand(0).getReg(); 11066 unsigned ptrA = MI.getOperand(1).getReg(); 11067 unsigned ptrB = MI.getOperand(2).getReg(); 11068 unsigned oldval = MI.getOperand(3).getReg(); 11069 unsigned newval = MI.getOperand(4).getReg(); 11070 DebugLoc dl = MI.getDebugLoc(); 11071 11072 MachineBasicBlock *loop1MBB = F->CreateMachineBasicBlock(LLVM_BB); 11073 MachineBasicBlock *loop2MBB = F->CreateMachineBasicBlock(LLVM_BB); 11074 MachineBasicBlock *midMBB = F->CreateMachineBasicBlock(LLVM_BB); 11075 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); 11076 F->insert(It, loop1MBB); 11077 F->insert(It, loop2MBB); 11078 F->insert(It, midMBB); 11079 F->insert(It, exitMBB); 11080 exitMBB->splice(exitMBB->begin(), BB, 11081 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 11082 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 11083 11084 MachineRegisterInfo &RegInfo = F->getRegInfo(); 11085 const TargetRegisterClass *RC = 11086 is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; 11087 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass; 11088 11089 Register PtrReg = RegInfo.createVirtualRegister(RC); 11090 Register Shift1Reg = RegInfo.createVirtualRegister(GPRC); 11091 Register ShiftReg = 11092 isLittleEndian ? Shift1Reg : RegInfo.createVirtualRegister(GPRC); 11093 Register NewVal2Reg = RegInfo.createVirtualRegister(GPRC); 11094 Register NewVal3Reg = RegInfo.createVirtualRegister(GPRC); 11095 Register OldVal2Reg = RegInfo.createVirtualRegister(GPRC); 11096 Register OldVal3Reg = RegInfo.createVirtualRegister(GPRC); 11097 Register MaskReg = RegInfo.createVirtualRegister(GPRC); 11098 Register Mask2Reg = RegInfo.createVirtualRegister(GPRC); 11099 Register Mask3Reg = RegInfo.createVirtualRegister(GPRC); 11100 Register Tmp2Reg = RegInfo.createVirtualRegister(GPRC); 11101 Register Tmp4Reg = RegInfo.createVirtualRegister(GPRC); 11102 Register TmpDestReg = RegInfo.createVirtualRegister(GPRC); 11103 Register Ptr1Reg; 11104 Register TmpReg = RegInfo.createVirtualRegister(GPRC); 11105 Register ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO; 11106 // thisMBB: 11107 // ... 11108 // fallthrough --> loopMBB 11109 BB->addSuccessor(loop1MBB); 11110 11111 // The 4-byte load must be aligned, while a char or short may be 11112 // anywhere in the word. Hence all this nasty bookkeeping code. 11113 // add ptr1, ptrA, ptrB [copy if ptrA==0] 11114 // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27] 11115 // xori shift, shift1, 24 [16] 11116 // rlwinm ptr, ptr1, 0, 0, 29 11117 // slw newval2, newval, shift 11118 // slw oldval2, oldval,shift 11119 // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535] 11120 // slw mask, mask2, shift 11121 // and newval3, newval2, mask 11122 // and oldval3, oldval2, mask 11123 // loop1MBB: 11124 // lwarx tmpDest, ptr 11125 // and tmp, tmpDest, mask 11126 // cmpw tmp, oldval3 11127 // bne- midMBB 11128 // loop2MBB: 11129 // andc tmp2, tmpDest, mask 11130 // or tmp4, tmp2, newval3 11131 // stwcx. tmp4, ptr 11132 // bne- loop1MBB 11133 // b exitBB 11134 // midMBB: 11135 // stwcx. tmpDest, ptr 11136 // exitBB: 11137 // srw dest, tmpDest, shift 11138 if (ptrA != ZeroReg) { 11139 Ptr1Reg = RegInfo.createVirtualRegister(RC); 11140 BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg) 11141 .addReg(ptrA) 11142 .addReg(ptrB); 11143 } else { 11144 Ptr1Reg = ptrB; 11145 } 11146 11147 // We need use 32-bit subregister to avoid mismatch register class in 64-bit 11148 // mode. 11149 BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg) 11150 .addReg(Ptr1Reg, 0, is64bit ? PPC::sub_32 : 0) 11151 .addImm(3) 11152 .addImm(27) 11153 .addImm(is8bit ? 28 : 27); 11154 if (!isLittleEndian) 11155 BuildMI(BB, dl, TII->get(PPC::XORI), ShiftReg) 11156 .addReg(Shift1Reg) 11157 .addImm(is8bit ? 24 : 16); 11158 if (is64bit) 11159 BuildMI(BB, dl, TII->get(PPC::RLDICR), PtrReg) 11160 .addReg(Ptr1Reg) 11161 .addImm(0) 11162 .addImm(61); 11163 else 11164 BuildMI(BB, dl, TII->get(PPC::RLWINM), PtrReg) 11165 .addReg(Ptr1Reg) 11166 .addImm(0) 11167 .addImm(0) 11168 .addImm(29); 11169 BuildMI(BB, dl, TII->get(PPC::SLW), NewVal2Reg) 11170 .addReg(newval) 11171 .addReg(ShiftReg); 11172 BuildMI(BB, dl, TII->get(PPC::SLW), OldVal2Reg) 11173 .addReg(oldval) 11174 .addReg(ShiftReg); 11175 if (is8bit) 11176 BuildMI(BB, dl, TII->get(PPC::LI), Mask2Reg).addImm(255); 11177 else { 11178 BuildMI(BB, dl, TII->get(PPC::LI), Mask3Reg).addImm(0); 11179 BuildMI(BB, dl, TII->get(PPC::ORI), Mask2Reg) 11180 .addReg(Mask3Reg) 11181 .addImm(65535); 11182 } 11183 BuildMI(BB, dl, TII->get(PPC::SLW), MaskReg) 11184 .addReg(Mask2Reg) 11185 .addReg(ShiftReg); 11186 BuildMI(BB, dl, TII->get(PPC::AND), NewVal3Reg) 11187 .addReg(NewVal2Reg) 11188 .addReg(MaskReg); 11189 BuildMI(BB, dl, TII->get(PPC::AND), OldVal3Reg) 11190 .addReg(OldVal2Reg) 11191 .addReg(MaskReg); 11192 11193 BB = loop1MBB; 11194 BuildMI(BB, dl, TII->get(PPC::LWARX), TmpDestReg) 11195 .addReg(ZeroReg) 11196 .addReg(PtrReg); 11197 BuildMI(BB, dl, TII->get(PPC::AND), TmpReg) 11198 .addReg(TmpDestReg) 11199 .addReg(MaskReg); 11200 BuildMI(BB, dl, TII->get(PPC::CMPW), PPC::CR0) 11201 .addReg(TmpReg) 11202 .addReg(OldVal3Reg); 11203 BuildMI(BB, dl, TII->get(PPC::BCC)) 11204 .addImm(PPC::PRED_NE) 11205 .addReg(PPC::CR0) 11206 .addMBB(midMBB); 11207 BB->addSuccessor(loop2MBB); 11208 BB->addSuccessor(midMBB); 11209 11210 BB = loop2MBB; 11211 BuildMI(BB, dl, TII->get(PPC::ANDC), Tmp2Reg) 11212 .addReg(TmpDestReg) 11213 .addReg(MaskReg); 11214 BuildMI(BB, dl, TII->get(PPC::OR), Tmp4Reg) 11215 .addReg(Tmp2Reg) 11216 .addReg(NewVal3Reg); 11217 BuildMI(BB, dl, TII->get(PPC::STWCX)) 11218 .addReg(Tmp4Reg) 11219 .addReg(ZeroReg) 11220 .addReg(PtrReg); 11221 BuildMI(BB, dl, TII->get(PPC::BCC)) 11222 .addImm(PPC::PRED_NE) 11223 .addReg(PPC::CR0) 11224 .addMBB(loop1MBB); 11225 BuildMI(BB, dl, TII->get(PPC::B)).addMBB(exitMBB); 11226 BB->addSuccessor(loop1MBB); 11227 BB->addSuccessor(exitMBB); 11228 11229 BB = midMBB; 11230 BuildMI(BB, dl, TII->get(PPC::STWCX)) 11231 .addReg(TmpDestReg) 11232 .addReg(ZeroReg) 11233 .addReg(PtrReg); 11234 BB->addSuccessor(exitMBB); 11235 11236 // exitMBB: 11237 // ... 11238 BB = exitMBB; 11239 BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), dest) 11240 .addReg(TmpReg) 11241 .addReg(ShiftReg); 11242 } else if (MI.getOpcode() == PPC::FADDrtz) { 11243 // This pseudo performs an FADD with rounding mode temporarily forced 11244 // to round-to-zero. We emit this via custom inserter since the FPSCR 11245 // is not modeled at the SelectionDAG level. 11246 unsigned Dest = MI.getOperand(0).getReg(); 11247 unsigned Src1 = MI.getOperand(1).getReg(); 11248 unsigned Src2 = MI.getOperand(2).getReg(); 11249 DebugLoc dl = MI.getDebugLoc(); 11250 11251 MachineRegisterInfo &RegInfo = F->getRegInfo(); 11252 unsigned MFFSReg = RegInfo.createVirtualRegister(&PPC::F8RCRegClass); 11253 11254 // Save FPSCR value. 11255 BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), MFFSReg); 11256 11257 // Set rounding mode to round-to-zero. 11258 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB1)).addImm(31); 11259 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB0)).addImm(30); 11260 11261 // Perform addition. 11262 BuildMI(*BB, MI, dl, TII->get(PPC::FADD), Dest).addReg(Src1).addReg(Src2); 11263 11264 // Restore FPSCR value. 11265 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSFb)).addImm(1).addReg(MFFSReg); 11266 } else if (MI.getOpcode() == PPC::ANDIo_1_EQ_BIT || 11267 MI.getOpcode() == PPC::ANDIo_1_GT_BIT || 11268 MI.getOpcode() == PPC::ANDIo_1_EQ_BIT8 || 11269 MI.getOpcode() == PPC::ANDIo_1_GT_BIT8) { 11270 unsigned Opcode = (MI.getOpcode() == PPC::ANDIo_1_EQ_BIT8 || 11271 MI.getOpcode() == PPC::ANDIo_1_GT_BIT8) 11272 ? PPC::ANDIo8 11273 : PPC::ANDIo; 11274 bool isEQ = (MI.getOpcode() == PPC::ANDIo_1_EQ_BIT || 11275 MI.getOpcode() == PPC::ANDIo_1_EQ_BIT8); 11276 11277 MachineRegisterInfo &RegInfo = F->getRegInfo(); 11278 unsigned Dest = RegInfo.createVirtualRegister( 11279 Opcode == PPC::ANDIo ? &PPC::GPRCRegClass : &PPC::G8RCRegClass); 11280 11281 DebugLoc dl = MI.getDebugLoc(); 11282 BuildMI(*BB, MI, dl, TII->get(Opcode), Dest) 11283 .addReg(MI.getOperand(1).getReg()) 11284 .addImm(1); 11285 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), 11286 MI.getOperand(0).getReg()) 11287 .addReg(isEQ ? PPC::CR0EQ : PPC::CR0GT); 11288 } else if (MI.getOpcode() == PPC::TCHECK_RET) { 11289 DebugLoc Dl = MI.getDebugLoc(); 11290 MachineRegisterInfo &RegInfo = F->getRegInfo(); 11291 unsigned CRReg = RegInfo.createVirtualRegister(&PPC::CRRCRegClass); 11292 BuildMI(*BB, MI, Dl, TII->get(PPC::TCHECK), CRReg); 11293 BuildMI(*BB, MI, Dl, TII->get(TargetOpcode::COPY), 11294 MI.getOperand(0).getReg()) 11295 .addReg(CRReg); 11296 } else if (MI.getOpcode() == PPC::TBEGIN_RET) { 11297 DebugLoc Dl = MI.getDebugLoc(); 11298 unsigned Imm = MI.getOperand(1).getImm(); 11299 BuildMI(*BB, MI, Dl, TII->get(PPC::TBEGIN)).addImm(Imm); 11300 BuildMI(*BB, MI, Dl, TII->get(TargetOpcode::COPY), 11301 MI.getOperand(0).getReg()) 11302 .addReg(PPC::CR0EQ); 11303 } else if (MI.getOpcode() == PPC::SETRNDi) { 11304 DebugLoc dl = MI.getDebugLoc(); 11305 unsigned OldFPSCRReg = MI.getOperand(0).getReg(); 11306 11307 // Save FPSCR value. 11308 BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), OldFPSCRReg); 11309 11310 // The floating point rounding mode is in the bits 62:63 of FPCSR, and has 11311 // the following settings: 11312 // 00 Round to nearest 11313 // 01 Round to 0 11314 // 10 Round to +inf 11315 // 11 Round to -inf 11316 11317 // When the operand is immediate, using the two least significant bits of 11318 // the immediate to set the bits 62:63 of FPSCR. 11319 unsigned Mode = MI.getOperand(1).getImm(); 11320 BuildMI(*BB, MI, dl, TII->get((Mode & 1) ? PPC::MTFSB1 : PPC::MTFSB0)) 11321 .addImm(31); 11322 11323 BuildMI(*BB, MI, dl, TII->get((Mode & 2) ? PPC::MTFSB1 : PPC::MTFSB0)) 11324 .addImm(30); 11325 } else if (MI.getOpcode() == PPC::SETRND) { 11326 DebugLoc dl = MI.getDebugLoc(); 11327 11328 // Copy register from F8RCRegClass::SrcReg to G8RCRegClass::DestReg 11329 // or copy register from G8RCRegClass::SrcReg to F8RCRegClass::DestReg. 11330 // If the target doesn't have DirectMove, we should use stack to do the 11331 // conversion, because the target doesn't have the instructions like mtvsrd 11332 // or mfvsrd to do this conversion directly. 11333 auto copyRegFromG8RCOrF8RC = [&] (unsigned DestReg, unsigned SrcReg) { 11334 if (Subtarget.hasDirectMove()) { 11335 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), DestReg) 11336 .addReg(SrcReg); 11337 } else { 11338 // Use stack to do the register copy. 11339 unsigned StoreOp = PPC::STD, LoadOp = PPC::LFD; 11340 MachineRegisterInfo &RegInfo = F->getRegInfo(); 11341 const TargetRegisterClass *RC = RegInfo.getRegClass(SrcReg); 11342 if (RC == &PPC::F8RCRegClass) { 11343 // Copy register from F8RCRegClass to G8RCRegclass. 11344 assert((RegInfo.getRegClass(DestReg) == &PPC::G8RCRegClass) && 11345 "Unsupported RegClass."); 11346 11347 StoreOp = PPC::STFD; 11348 LoadOp = PPC::LD; 11349 } else { 11350 // Copy register from G8RCRegClass to F8RCRegclass. 11351 assert((RegInfo.getRegClass(SrcReg) == &PPC::G8RCRegClass) && 11352 (RegInfo.getRegClass(DestReg) == &PPC::F8RCRegClass) && 11353 "Unsupported RegClass."); 11354 } 11355 11356 MachineFrameInfo &MFI = F->getFrameInfo(); 11357 int FrameIdx = MFI.CreateStackObject(8, 8, false); 11358 11359 MachineMemOperand *MMOStore = F->getMachineMemOperand( 11360 MachinePointerInfo::getFixedStack(*F, FrameIdx, 0), 11361 MachineMemOperand::MOStore, MFI.getObjectSize(FrameIdx), 11362 MFI.getObjectAlignment(FrameIdx)); 11363 11364 // Store the SrcReg into the stack. 11365 BuildMI(*BB, MI, dl, TII->get(StoreOp)) 11366 .addReg(SrcReg) 11367 .addImm(0) 11368 .addFrameIndex(FrameIdx) 11369 .addMemOperand(MMOStore); 11370 11371 MachineMemOperand *MMOLoad = F->getMachineMemOperand( 11372 MachinePointerInfo::getFixedStack(*F, FrameIdx, 0), 11373 MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIdx), 11374 MFI.getObjectAlignment(FrameIdx)); 11375 11376 // Load from the stack where SrcReg is stored, and save to DestReg, 11377 // so we have done the RegClass conversion from RegClass::SrcReg to 11378 // RegClass::DestReg. 11379 BuildMI(*BB, MI, dl, TII->get(LoadOp), DestReg) 11380 .addImm(0) 11381 .addFrameIndex(FrameIdx) 11382 .addMemOperand(MMOLoad); 11383 } 11384 }; 11385 11386 unsigned OldFPSCRReg = MI.getOperand(0).getReg(); 11387 11388 // Save FPSCR value. 11389 BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), OldFPSCRReg); 11390 11391 // When the operand is gprc register, use two least significant bits of the 11392 // register and mtfsf instruction to set the bits 62:63 of FPSCR. 11393 // 11394 // copy OldFPSCRTmpReg, OldFPSCRReg 11395 // (INSERT_SUBREG ExtSrcReg, (IMPLICIT_DEF ImDefReg), SrcOp, 1) 11396 // rldimi NewFPSCRTmpReg, ExtSrcReg, OldFPSCRReg, 0, 62 11397 // copy NewFPSCRReg, NewFPSCRTmpReg 11398 // mtfsf 255, NewFPSCRReg 11399 MachineOperand SrcOp = MI.getOperand(1); 11400 MachineRegisterInfo &RegInfo = F->getRegInfo(); 11401 unsigned OldFPSCRTmpReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass); 11402 11403 copyRegFromG8RCOrF8RC(OldFPSCRTmpReg, OldFPSCRReg); 11404 11405 unsigned ImDefReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass); 11406 unsigned ExtSrcReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass); 11407 11408 // The first operand of INSERT_SUBREG should be a register which has 11409 // subregisters, we only care about its RegClass, so we should use an 11410 // IMPLICIT_DEF register. 11411 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::IMPLICIT_DEF), ImDefReg); 11412 BuildMI(*BB, MI, dl, TII->get(PPC::INSERT_SUBREG), ExtSrcReg) 11413 .addReg(ImDefReg) 11414 .add(SrcOp) 11415 .addImm(1); 11416 11417 unsigned NewFPSCRTmpReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass); 11418 BuildMI(*BB, MI, dl, TII->get(PPC::RLDIMI), NewFPSCRTmpReg) 11419 .addReg(OldFPSCRTmpReg) 11420 .addReg(ExtSrcReg) 11421 .addImm(0) 11422 .addImm(62); 11423 11424 unsigned NewFPSCRReg = RegInfo.createVirtualRegister(&PPC::F8RCRegClass); 11425 copyRegFromG8RCOrF8RC(NewFPSCRReg, NewFPSCRTmpReg); 11426 11427 // The mask 255 means that put the 32:63 bits of NewFPSCRReg to the 32:63 11428 // bits of FPSCR. 11429 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSF)) 11430 .addImm(255) 11431 .addReg(NewFPSCRReg) 11432 .addImm(0) 11433 .addImm(0); 11434 } else { 11435 llvm_unreachable("Unexpected instr type to insert"); 11436 } 11437 11438 MI.eraseFromParent(); // The pseudo instruction is gone now. 11439 return BB; 11440 } 11441 11442 //===----------------------------------------------------------------------===// 11443 // Target Optimization Hooks 11444 //===----------------------------------------------------------------------===// 11445 11446 static int getEstimateRefinementSteps(EVT VT, const PPCSubtarget &Subtarget) { 11447 // For the estimates, convergence is quadratic, so we essentially double the 11448 // number of digits correct after every iteration. For both FRE and FRSQRTE, 11449 // the minimum architected relative accuracy is 2^-5. When hasRecipPrec(), 11450 // this is 2^-14. IEEE float has 23 digits and double has 52 digits. 11451 int RefinementSteps = Subtarget.hasRecipPrec() ? 1 : 3; 11452 if (VT.getScalarType() == MVT::f64) 11453 RefinementSteps++; 11454 return RefinementSteps; 11455 } 11456 11457 SDValue PPCTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, 11458 int Enabled, int &RefinementSteps, 11459 bool &UseOneConstNR, 11460 bool Reciprocal) const { 11461 EVT VT = Operand.getValueType(); 11462 if ((VT == MVT::f32 && Subtarget.hasFRSQRTES()) || 11463 (VT == MVT::f64 && Subtarget.hasFRSQRTE()) || 11464 (VT == MVT::v4f32 && Subtarget.hasAltivec()) || 11465 (VT == MVT::v2f64 && Subtarget.hasVSX()) || 11466 (VT == MVT::v4f32 && Subtarget.hasQPX()) || 11467 (VT == MVT::v4f64 && Subtarget.hasQPX())) { 11468 if (RefinementSteps == ReciprocalEstimate::Unspecified) 11469 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget); 11470 11471 // The Newton-Raphson computation with a single constant does not provide 11472 // enough accuracy on some CPUs. 11473 UseOneConstNR = !Subtarget.needsTwoConstNR(); 11474 return DAG.getNode(PPCISD::FRSQRTE, SDLoc(Operand), VT, Operand); 11475 } 11476 return SDValue(); 11477 } 11478 11479 SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand, SelectionDAG &DAG, 11480 int Enabled, 11481 int &RefinementSteps) const { 11482 EVT VT = Operand.getValueType(); 11483 if ((VT == MVT::f32 && Subtarget.hasFRES()) || 11484 (VT == MVT::f64 && Subtarget.hasFRE()) || 11485 (VT == MVT::v4f32 && Subtarget.hasAltivec()) || 11486 (VT == MVT::v2f64 && Subtarget.hasVSX()) || 11487 (VT == MVT::v4f32 && Subtarget.hasQPX()) || 11488 (VT == MVT::v4f64 && Subtarget.hasQPX())) { 11489 if (RefinementSteps == ReciprocalEstimate::Unspecified) 11490 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget); 11491 return DAG.getNode(PPCISD::FRE, SDLoc(Operand), VT, Operand); 11492 } 11493 return SDValue(); 11494 } 11495 11496 unsigned PPCTargetLowering::combineRepeatedFPDivisors() const { 11497 // Note: This functionality is used only when unsafe-fp-math is enabled, and 11498 // on cores with reciprocal estimates (which are used when unsafe-fp-math is 11499 // enabled for division), this functionality is redundant with the default 11500 // combiner logic (once the division -> reciprocal/multiply transformation 11501 // has taken place). As a result, this matters more for older cores than for 11502 // newer ones. 11503 11504 // Combine multiple FDIVs with the same divisor into multiple FMULs by the 11505 // reciprocal if there are two or more FDIVs (for embedded cores with only 11506 // one FP pipeline) for three or more FDIVs (for generic OOO cores). 11507 switch (Subtarget.getDarwinDirective()) { 11508 default: 11509 return 3; 11510 case PPC::DIR_440: 11511 case PPC::DIR_A2: 11512 case PPC::DIR_E500: 11513 case PPC::DIR_E500mc: 11514 case PPC::DIR_E5500: 11515 return 2; 11516 } 11517 } 11518 11519 // isConsecutiveLSLoc needs to work even if all adds have not yet been 11520 // collapsed, and so we need to look through chains of them. 11521 static void getBaseWithConstantOffset(SDValue Loc, SDValue &Base, 11522 int64_t& Offset, SelectionDAG &DAG) { 11523 if (DAG.isBaseWithConstantOffset(Loc)) { 11524 Base = Loc.getOperand(0); 11525 Offset += cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue(); 11526 11527 // The base might itself be a base plus an offset, and if so, accumulate 11528 // that as well. 11529 getBaseWithConstantOffset(Loc.getOperand(0), Base, Offset, DAG); 11530 } 11531 } 11532 11533 static bool isConsecutiveLSLoc(SDValue Loc, EVT VT, LSBaseSDNode *Base, 11534 unsigned Bytes, int Dist, 11535 SelectionDAG &DAG) { 11536 if (VT.getSizeInBits() / 8 != Bytes) 11537 return false; 11538 11539 SDValue BaseLoc = Base->getBasePtr(); 11540 if (Loc.getOpcode() == ISD::FrameIndex) { 11541 if (BaseLoc.getOpcode() != ISD::FrameIndex) 11542 return false; 11543 const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); 11544 int FI = cast<FrameIndexSDNode>(Loc)->getIndex(); 11545 int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex(); 11546 int FS = MFI.getObjectSize(FI); 11547 int BFS = MFI.getObjectSize(BFI); 11548 if (FS != BFS || FS != (int)Bytes) return false; 11549 return MFI.getObjectOffset(FI) == (MFI.getObjectOffset(BFI) + Dist*Bytes); 11550 } 11551 11552 SDValue Base1 = Loc, Base2 = BaseLoc; 11553 int64_t Offset1 = 0, Offset2 = 0; 11554 getBaseWithConstantOffset(Loc, Base1, Offset1, DAG); 11555 getBaseWithConstantOffset(BaseLoc, Base2, Offset2, DAG); 11556 if (Base1 == Base2 && Offset1 == (Offset2 + Dist * Bytes)) 11557 return true; 11558 11559 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 11560 const GlobalValue *GV1 = nullptr; 11561 const GlobalValue *GV2 = nullptr; 11562 Offset1 = 0; 11563 Offset2 = 0; 11564 bool isGA1 = TLI.isGAPlusOffset(Loc.getNode(), GV1, Offset1); 11565 bool isGA2 = TLI.isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2); 11566 if (isGA1 && isGA2 && GV1 == GV2) 11567 return Offset1 == (Offset2 + Dist*Bytes); 11568 return false; 11569 } 11570 11571 // Like SelectionDAG::isConsecutiveLoad, but also works for stores, and does 11572 // not enforce equality of the chain operands. 11573 static bool isConsecutiveLS(SDNode *N, LSBaseSDNode *Base, 11574 unsigned Bytes, int Dist, 11575 SelectionDAG &DAG) { 11576 if (LSBaseSDNode *LS = dyn_cast<LSBaseSDNode>(N)) { 11577 EVT VT = LS->getMemoryVT(); 11578 SDValue Loc = LS->getBasePtr(); 11579 return isConsecutiveLSLoc(Loc, VT, Base, Bytes, Dist, DAG); 11580 } 11581 11582 if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) { 11583 EVT VT; 11584 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) { 11585 default: return false; 11586 case Intrinsic::ppc_qpx_qvlfd: 11587 case Intrinsic::ppc_qpx_qvlfda: 11588 VT = MVT::v4f64; 11589 break; 11590 case Intrinsic::ppc_qpx_qvlfs: 11591 case Intrinsic::ppc_qpx_qvlfsa: 11592 VT = MVT::v4f32; 11593 break; 11594 case Intrinsic::ppc_qpx_qvlfcd: 11595 case Intrinsic::ppc_qpx_qvlfcda: 11596 VT = MVT::v2f64; 11597 break; 11598 case Intrinsic::ppc_qpx_qvlfcs: 11599 case Intrinsic::ppc_qpx_qvlfcsa: 11600 VT = MVT::v2f32; 11601 break; 11602 case Intrinsic::ppc_qpx_qvlfiwa: 11603 case Intrinsic::ppc_qpx_qvlfiwz: 11604 case Intrinsic::ppc_altivec_lvx: 11605 case Intrinsic::ppc_altivec_lvxl: 11606 case Intrinsic::ppc_vsx_lxvw4x: 11607 case Intrinsic::ppc_vsx_lxvw4x_be: 11608 VT = MVT::v4i32; 11609 break; 11610 case Intrinsic::ppc_vsx_lxvd2x: 11611 case Intrinsic::ppc_vsx_lxvd2x_be: 11612 VT = MVT::v2f64; 11613 break; 11614 case Intrinsic::ppc_altivec_lvebx: 11615 VT = MVT::i8; 11616 break; 11617 case Intrinsic::ppc_altivec_lvehx: 11618 VT = MVT::i16; 11619 break; 11620 case Intrinsic::ppc_altivec_lvewx: 11621 VT = MVT::i32; 11622 break; 11623 } 11624 11625 return isConsecutiveLSLoc(N->getOperand(2), VT, Base, Bytes, Dist, DAG); 11626 } 11627 11628 if (N->getOpcode() == ISD::INTRINSIC_VOID) { 11629 EVT VT; 11630 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) { 11631 default: return false; 11632 case Intrinsic::ppc_qpx_qvstfd: 11633 case Intrinsic::ppc_qpx_qvstfda: 11634 VT = MVT::v4f64; 11635 break; 11636 case Intrinsic::ppc_qpx_qvstfs: 11637 case Intrinsic::ppc_qpx_qvstfsa: 11638 VT = MVT::v4f32; 11639 break; 11640 case Intrinsic::ppc_qpx_qvstfcd: 11641 case Intrinsic::ppc_qpx_qvstfcda: 11642 VT = MVT::v2f64; 11643 break; 11644 case Intrinsic::ppc_qpx_qvstfcs: 11645 case Intrinsic::ppc_qpx_qvstfcsa: 11646 VT = MVT::v2f32; 11647 break; 11648 case Intrinsic::ppc_qpx_qvstfiw: 11649 case Intrinsic::ppc_qpx_qvstfiwa: 11650 case Intrinsic::ppc_altivec_stvx: 11651 case Intrinsic::ppc_altivec_stvxl: 11652 case Intrinsic::ppc_vsx_stxvw4x: 11653 VT = MVT::v4i32; 11654 break; 11655 case Intrinsic::ppc_vsx_stxvd2x: 11656 VT = MVT::v2f64; 11657 break; 11658 case Intrinsic::ppc_vsx_stxvw4x_be: 11659 VT = MVT::v4i32; 11660 break; 11661 case Intrinsic::ppc_vsx_stxvd2x_be: 11662 VT = MVT::v2f64; 11663 break; 11664 case Intrinsic::ppc_altivec_stvebx: 11665 VT = MVT::i8; 11666 break; 11667 case Intrinsic::ppc_altivec_stvehx: 11668 VT = MVT::i16; 11669 break; 11670 case Intrinsic::ppc_altivec_stvewx: 11671 VT = MVT::i32; 11672 break; 11673 } 11674 11675 return isConsecutiveLSLoc(N->getOperand(3), VT, Base, Bytes, Dist, DAG); 11676 } 11677 11678 return false; 11679 } 11680 11681 // Return true is there is a nearyby consecutive load to the one provided 11682 // (regardless of alignment). We search up and down the chain, looking though 11683 // token factors and other loads (but nothing else). As a result, a true result 11684 // indicates that it is safe to create a new consecutive load adjacent to the 11685 // load provided. 11686 static bool findConsecutiveLoad(LoadSDNode *LD, SelectionDAG &DAG) { 11687 SDValue Chain = LD->getChain(); 11688 EVT VT = LD->getMemoryVT(); 11689 11690 SmallSet<SDNode *, 16> LoadRoots; 11691 SmallVector<SDNode *, 8> Queue(1, Chain.getNode()); 11692 SmallSet<SDNode *, 16> Visited; 11693 11694 // First, search up the chain, branching to follow all token-factor operands. 11695 // If we find a consecutive load, then we're done, otherwise, record all 11696 // nodes just above the top-level loads and token factors. 11697 while (!Queue.empty()) { 11698 SDNode *ChainNext = Queue.pop_back_val(); 11699 if (!Visited.insert(ChainNext).second) 11700 continue; 11701 11702 if (MemSDNode *ChainLD = dyn_cast<MemSDNode>(ChainNext)) { 11703 if (isConsecutiveLS(ChainLD, LD, VT.getStoreSize(), 1, DAG)) 11704 return true; 11705 11706 if (!Visited.count(ChainLD->getChain().getNode())) 11707 Queue.push_back(ChainLD->getChain().getNode()); 11708 } else if (ChainNext->getOpcode() == ISD::TokenFactor) { 11709 for (const SDUse &O : ChainNext->ops()) 11710 if (!Visited.count(O.getNode())) 11711 Queue.push_back(O.getNode()); 11712 } else 11713 LoadRoots.insert(ChainNext); 11714 } 11715 11716 // Second, search down the chain, starting from the top-level nodes recorded 11717 // in the first phase. These top-level nodes are the nodes just above all 11718 // loads and token factors. Starting with their uses, recursively look though 11719 // all loads (just the chain uses) and token factors to find a consecutive 11720 // load. 11721 Visited.clear(); 11722 Queue.clear(); 11723 11724 for (SmallSet<SDNode *, 16>::iterator I = LoadRoots.begin(), 11725 IE = LoadRoots.end(); I != IE; ++I) { 11726 Queue.push_back(*I); 11727 11728 while (!Queue.empty()) { 11729 SDNode *LoadRoot = Queue.pop_back_val(); 11730 if (!Visited.insert(LoadRoot).second) 11731 continue; 11732 11733 if (MemSDNode *ChainLD = dyn_cast<MemSDNode>(LoadRoot)) 11734 if (isConsecutiveLS(ChainLD, LD, VT.getStoreSize(), 1, DAG)) 11735 return true; 11736 11737 for (SDNode::use_iterator UI = LoadRoot->use_begin(), 11738 UE = LoadRoot->use_end(); UI != UE; ++UI) 11739 if (((isa<MemSDNode>(*UI) && 11740 cast<MemSDNode>(*UI)->getChain().getNode() == LoadRoot) || 11741 UI->getOpcode() == ISD::TokenFactor) && !Visited.count(*UI)) 11742 Queue.push_back(*UI); 11743 } 11744 } 11745 11746 return false; 11747 } 11748 11749 /// This function is called when we have proved that a SETCC node can be replaced 11750 /// by subtraction (and other supporting instructions) so that the result of 11751 /// comparison is kept in a GPR instead of CR. This function is purely for 11752 /// codegen purposes and has some flags to guide the codegen process. 11753 static SDValue generateEquivalentSub(SDNode *N, int Size, bool Complement, 11754 bool Swap, SDLoc &DL, SelectionDAG &DAG) { 11755 assert(N->getOpcode() == ISD::SETCC && "ISD::SETCC Expected."); 11756 11757 // Zero extend the operands to the largest legal integer. Originally, they 11758 // must be of a strictly smaller size. 11759 auto Op0 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(0), 11760 DAG.getConstant(Size, DL, MVT::i32)); 11761 auto Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1), 11762 DAG.getConstant(Size, DL, MVT::i32)); 11763 11764 // Swap if needed. Depends on the condition code. 11765 if (Swap) 11766 std::swap(Op0, Op1); 11767 11768 // Subtract extended integers. 11769 auto SubNode = DAG.getNode(ISD::SUB, DL, MVT::i64, Op0, Op1); 11770 11771 // Move the sign bit to the least significant position and zero out the rest. 11772 // Now the least significant bit carries the result of original comparison. 11773 auto Shifted = DAG.getNode(ISD::SRL, DL, MVT::i64, SubNode, 11774 DAG.getConstant(Size - 1, DL, MVT::i32)); 11775 auto Final = Shifted; 11776 11777 // Complement the result if needed. Based on the condition code. 11778 if (Complement) 11779 Final = DAG.getNode(ISD::XOR, DL, MVT::i64, Shifted, 11780 DAG.getConstant(1, DL, MVT::i64)); 11781 11782 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Final); 11783 } 11784 11785 SDValue PPCTargetLowering::ConvertSETCCToSubtract(SDNode *N, 11786 DAGCombinerInfo &DCI) const { 11787 assert(N->getOpcode() == ISD::SETCC && "ISD::SETCC Expected."); 11788 11789 SelectionDAG &DAG = DCI.DAG; 11790 SDLoc DL(N); 11791 11792 // Size of integers being compared has a critical role in the following 11793 // analysis, so we prefer to do this when all types are legal. 11794 if (!DCI.isAfterLegalizeDAG()) 11795 return SDValue(); 11796 11797 // If all users of SETCC extend its value to a legal integer type 11798 // then we replace SETCC with a subtraction 11799 for (SDNode::use_iterator UI = N->use_begin(), 11800 UE = N->use_end(); UI != UE; ++UI) { 11801 if (UI->getOpcode() != ISD::ZERO_EXTEND) 11802 return SDValue(); 11803 } 11804 11805 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get(); 11806 auto OpSize = N->getOperand(0).getValueSizeInBits(); 11807 11808 unsigned Size = DAG.getDataLayout().getLargestLegalIntTypeSizeInBits(); 11809 11810 if (OpSize < Size) { 11811 switch (CC) { 11812 default: break; 11813 case ISD::SETULT: 11814 return generateEquivalentSub(N, Size, false, false, DL, DAG); 11815 case ISD::SETULE: 11816 return generateEquivalentSub(N, Size, true, true, DL, DAG); 11817 case ISD::SETUGT: 11818 return generateEquivalentSub(N, Size, false, true, DL, DAG); 11819 case ISD::SETUGE: 11820 return generateEquivalentSub(N, Size, true, false, DL, DAG); 11821 } 11822 } 11823 11824 return SDValue(); 11825 } 11826 11827 SDValue PPCTargetLowering::DAGCombineTruncBoolExt(SDNode *N, 11828 DAGCombinerInfo &DCI) const { 11829 SelectionDAG &DAG = DCI.DAG; 11830 SDLoc dl(N); 11831 11832 assert(Subtarget.useCRBits() && "Expecting to be tracking CR bits"); 11833 // If we're tracking CR bits, we need to be careful that we don't have: 11834 // trunc(binary-ops(zext(x), zext(y))) 11835 // or 11836 // trunc(binary-ops(binary-ops(zext(x), zext(y)), ...) 11837 // such that we're unnecessarily moving things into GPRs when it would be 11838 // better to keep them in CR bits. 11839 11840 // Note that trunc here can be an actual i1 trunc, or can be the effective 11841 // truncation that comes from a setcc or select_cc. 11842 if (N->getOpcode() == ISD::TRUNCATE && 11843 N->getValueType(0) != MVT::i1) 11844 return SDValue(); 11845 11846 if (N->getOperand(0).getValueType() != MVT::i32 && 11847 N->getOperand(0).getValueType() != MVT::i64) 11848 return SDValue(); 11849 11850 if (N->getOpcode() == ISD::SETCC || 11851 N->getOpcode() == ISD::SELECT_CC) { 11852 // If we're looking at a comparison, then we need to make sure that the 11853 // high bits (all except for the first) don't matter the result. 11854 ISD::CondCode CC = 11855 cast<CondCodeSDNode>(N->getOperand( 11856 N->getOpcode() == ISD::SETCC ? 2 : 4))->get(); 11857 unsigned OpBits = N->getOperand(0).getValueSizeInBits(); 11858 11859 if (ISD::isSignedIntSetCC(CC)) { 11860 if (DAG.ComputeNumSignBits(N->getOperand(0)) != OpBits || 11861 DAG.ComputeNumSignBits(N->getOperand(1)) != OpBits) 11862 return SDValue(); 11863 } else if (ISD::isUnsignedIntSetCC(CC)) { 11864 if (!DAG.MaskedValueIsZero(N->getOperand(0), 11865 APInt::getHighBitsSet(OpBits, OpBits-1)) || 11866 !DAG.MaskedValueIsZero(N->getOperand(1), 11867 APInt::getHighBitsSet(OpBits, OpBits-1))) 11868 return (N->getOpcode() == ISD::SETCC ? ConvertSETCCToSubtract(N, DCI) 11869 : SDValue()); 11870 } else { 11871 // This is neither a signed nor an unsigned comparison, just make sure 11872 // that the high bits are equal. 11873 KnownBits Op1Known = DAG.computeKnownBits(N->getOperand(0)); 11874 KnownBits Op2Known = DAG.computeKnownBits(N->getOperand(1)); 11875 11876 // We don't really care about what is known about the first bit (if 11877 // anything), so clear it in all masks prior to comparing them. 11878 Op1Known.Zero.clearBit(0); Op1Known.One.clearBit(0); 11879 Op2Known.Zero.clearBit(0); Op2Known.One.clearBit(0); 11880 11881 if (Op1Known.Zero != Op2Known.Zero || Op1Known.One != Op2Known.One) 11882 return SDValue(); 11883 } 11884 } 11885 11886 // We now know that the higher-order bits are irrelevant, we just need to 11887 // make sure that all of the intermediate operations are bit operations, and 11888 // all inputs are extensions. 11889 if (N->getOperand(0).getOpcode() != ISD::AND && 11890 N->getOperand(0).getOpcode() != ISD::OR && 11891 N->getOperand(0).getOpcode() != ISD::XOR && 11892 N->getOperand(0).getOpcode() != ISD::SELECT && 11893 N->getOperand(0).getOpcode() != ISD::SELECT_CC && 11894 N->getOperand(0).getOpcode() != ISD::TRUNCATE && 11895 N->getOperand(0).getOpcode() != ISD::SIGN_EXTEND && 11896 N->getOperand(0).getOpcode() != ISD::ZERO_EXTEND && 11897 N->getOperand(0).getOpcode() != ISD::ANY_EXTEND) 11898 return SDValue(); 11899 11900 if ((N->getOpcode() == ISD::SETCC || N->getOpcode() == ISD::SELECT_CC) && 11901 N->getOperand(1).getOpcode() != ISD::AND && 11902 N->getOperand(1).getOpcode() != ISD::OR && 11903 N->getOperand(1).getOpcode() != ISD::XOR && 11904 N->getOperand(1).getOpcode() != ISD::SELECT && 11905 N->getOperand(1).getOpcode() != ISD::SELECT_CC && 11906 N->getOperand(1).getOpcode() != ISD::TRUNCATE && 11907 N->getOperand(1).getOpcode() != ISD::SIGN_EXTEND && 11908 N->getOperand(1).getOpcode() != ISD::ZERO_EXTEND && 11909 N->getOperand(1).getOpcode() != ISD::ANY_EXTEND) 11910 return SDValue(); 11911 11912 SmallVector<SDValue, 4> Inputs; 11913 SmallVector<SDValue, 8> BinOps, PromOps; 11914 SmallPtrSet<SDNode *, 16> Visited; 11915 11916 for (unsigned i = 0; i < 2; ++i) { 11917 if (((N->getOperand(i).getOpcode() == ISD::SIGN_EXTEND || 11918 N->getOperand(i).getOpcode() == ISD::ZERO_EXTEND || 11919 N->getOperand(i).getOpcode() == ISD::ANY_EXTEND) && 11920 N->getOperand(i).getOperand(0).getValueType() == MVT::i1) || 11921 isa<ConstantSDNode>(N->getOperand(i))) 11922 Inputs.push_back(N->getOperand(i)); 11923 else 11924 BinOps.push_back(N->getOperand(i)); 11925 11926 if (N->getOpcode() == ISD::TRUNCATE) 11927 break; 11928 } 11929 11930 // Visit all inputs, collect all binary operations (and, or, xor and 11931 // select) that are all fed by extensions. 11932 while (!BinOps.empty()) { 11933 SDValue BinOp = BinOps.back(); 11934 BinOps.pop_back(); 11935 11936 if (!Visited.insert(BinOp.getNode()).second) 11937 continue; 11938 11939 PromOps.push_back(BinOp); 11940 11941 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) { 11942 // The condition of the select is not promoted. 11943 if (BinOp.getOpcode() == ISD::SELECT && i == 0) 11944 continue; 11945 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3) 11946 continue; 11947 11948 if (((BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND || 11949 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND || 11950 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) && 11951 BinOp.getOperand(i).getOperand(0).getValueType() == MVT::i1) || 11952 isa<ConstantSDNode>(BinOp.getOperand(i))) { 11953 Inputs.push_back(BinOp.getOperand(i)); 11954 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND || 11955 BinOp.getOperand(i).getOpcode() == ISD::OR || 11956 BinOp.getOperand(i).getOpcode() == ISD::XOR || 11957 BinOp.getOperand(i).getOpcode() == ISD::SELECT || 11958 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC || 11959 BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE || 11960 BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND || 11961 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND || 11962 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) { 11963 BinOps.push_back(BinOp.getOperand(i)); 11964 } else { 11965 // We have an input that is not an extension or another binary 11966 // operation; we'll abort this transformation. 11967 return SDValue(); 11968 } 11969 } 11970 } 11971 11972 // Make sure that this is a self-contained cluster of operations (which 11973 // is not quite the same thing as saying that everything has only one 11974 // use). 11975 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 11976 if (isa<ConstantSDNode>(Inputs[i])) 11977 continue; 11978 11979 for (SDNode::use_iterator UI = Inputs[i].getNode()->use_begin(), 11980 UE = Inputs[i].getNode()->use_end(); 11981 UI != UE; ++UI) { 11982 SDNode *User = *UI; 11983 if (User != N && !Visited.count(User)) 11984 return SDValue(); 11985 11986 // Make sure that we're not going to promote the non-output-value 11987 // operand(s) or SELECT or SELECT_CC. 11988 // FIXME: Although we could sometimes handle this, and it does occur in 11989 // practice that one of the condition inputs to the select is also one of 11990 // the outputs, we currently can't deal with this. 11991 if (User->getOpcode() == ISD::SELECT) { 11992 if (User->getOperand(0) == Inputs[i]) 11993 return SDValue(); 11994 } else if (User->getOpcode() == ISD::SELECT_CC) { 11995 if (User->getOperand(0) == Inputs[i] || 11996 User->getOperand(1) == Inputs[i]) 11997 return SDValue(); 11998 } 11999 } 12000 } 12001 12002 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) { 12003 for (SDNode::use_iterator UI = PromOps[i].getNode()->use_begin(), 12004 UE = PromOps[i].getNode()->use_end(); 12005 UI != UE; ++UI) { 12006 SDNode *User = *UI; 12007 if (User != N && !Visited.count(User)) 12008 return SDValue(); 12009 12010 // Make sure that we're not going to promote the non-output-value 12011 // operand(s) or SELECT or SELECT_CC. 12012 // FIXME: Although we could sometimes handle this, and it does occur in 12013 // practice that one of the condition inputs to the select is also one of 12014 // the outputs, we currently can't deal with this. 12015 if (User->getOpcode() == ISD::SELECT) { 12016 if (User->getOperand(0) == PromOps[i]) 12017 return SDValue(); 12018 } else if (User->getOpcode() == ISD::SELECT_CC) { 12019 if (User->getOperand(0) == PromOps[i] || 12020 User->getOperand(1) == PromOps[i]) 12021 return SDValue(); 12022 } 12023 } 12024 } 12025 12026 // Replace all inputs with the extension operand. 12027 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 12028 // Constants may have users outside the cluster of to-be-promoted nodes, 12029 // and so we need to replace those as we do the promotions. 12030 if (isa<ConstantSDNode>(Inputs[i])) 12031 continue; 12032 else 12033 DAG.ReplaceAllUsesOfValueWith(Inputs[i], Inputs[i].getOperand(0)); 12034 } 12035 12036 std::list<HandleSDNode> PromOpHandles; 12037 for (auto &PromOp : PromOps) 12038 PromOpHandles.emplace_back(PromOp); 12039 12040 // Replace all operations (these are all the same, but have a different 12041 // (i1) return type). DAG.getNode will validate that the types of 12042 // a binary operator match, so go through the list in reverse so that 12043 // we've likely promoted both operands first. Any intermediate truncations or 12044 // extensions disappear. 12045 while (!PromOpHandles.empty()) { 12046 SDValue PromOp = PromOpHandles.back().getValue(); 12047 PromOpHandles.pop_back(); 12048 12049 if (PromOp.getOpcode() == ISD::TRUNCATE || 12050 PromOp.getOpcode() == ISD::SIGN_EXTEND || 12051 PromOp.getOpcode() == ISD::ZERO_EXTEND || 12052 PromOp.getOpcode() == ISD::ANY_EXTEND) { 12053 if (!isa<ConstantSDNode>(PromOp.getOperand(0)) && 12054 PromOp.getOperand(0).getValueType() != MVT::i1) { 12055 // The operand is not yet ready (see comment below). 12056 PromOpHandles.emplace_front(PromOp); 12057 continue; 12058 } 12059 12060 SDValue RepValue = PromOp.getOperand(0); 12061 if (isa<ConstantSDNode>(RepValue)) 12062 RepValue = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, RepValue); 12063 12064 DAG.ReplaceAllUsesOfValueWith(PromOp, RepValue); 12065 continue; 12066 } 12067 12068 unsigned C; 12069 switch (PromOp.getOpcode()) { 12070 default: C = 0; break; 12071 case ISD::SELECT: C = 1; break; 12072 case ISD::SELECT_CC: C = 2; break; 12073 } 12074 12075 if ((!isa<ConstantSDNode>(PromOp.getOperand(C)) && 12076 PromOp.getOperand(C).getValueType() != MVT::i1) || 12077 (!isa<ConstantSDNode>(PromOp.getOperand(C+1)) && 12078 PromOp.getOperand(C+1).getValueType() != MVT::i1)) { 12079 // The to-be-promoted operands of this node have not yet been 12080 // promoted (this should be rare because we're going through the 12081 // list backward, but if one of the operands has several users in 12082 // this cluster of to-be-promoted nodes, it is possible). 12083 PromOpHandles.emplace_front(PromOp); 12084 continue; 12085 } 12086 12087 SmallVector<SDValue, 3> Ops(PromOp.getNode()->op_begin(), 12088 PromOp.getNode()->op_end()); 12089 12090 // If there are any constant inputs, make sure they're replaced now. 12091 for (unsigned i = 0; i < 2; ++i) 12092 if (isa<ConstantSDNode>(Ops[C+i])) 12093 Ops[C+i] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Ops[C+i]); 12094 12095 DAG.ReplaceAllUsesOfValueWith(PromOp, 12096 DAG.getNode(PromOp.getOpcode(), dl, MVT::i1, Ops)); 12097 } 12098 12099 // Now we're left with the initial truncation itself. 12100 if (N->getOpcode() == ISD::TRUNCATE) 12101 return N->getOperand(0); 12102 12103 // Otherwise, this is a comparison. The operands to be compared have just 12104 // changed type (to i1), but everything else is the same. 12105 return SDValue(N, 0); 12106 } 12107 12108 SDValue PPCTargetLowering::DAGCombineExtBoolTrunc(SDNode *N, 12109 DAGCombinerInfo &DCI) const { 12110 SelectionDAG &DAG = DCI.DAG; 12111 SDLoc dl(N); 12112 12113 // If we're tracking CR bits, we need to be careful that we don't have: 12114 // zext(binary-ops(trunc(x), trunc(y))) 12115 // or 12116 // zext(binary-ops(binary-ops(trunc(x), trunc(y)), ...) 12117 // such that we're unnecessarily moving things into CR bits that can more 12118 // efficiently stay in GPRs. Note that if we're not certain that the high 12119 // bits are set as required by the final extension, we still may need to do 12120 // some masking to get the proper behavior. 12121 12122 // This same functionality is important on PPC64 when dealing with 12123 // 32-to-64-bit extensions; these occur often when 32-bit values are used as 12124 // the return values of functions. Because it is so similar, it is handled 12125 // here as well. 12126 12127 if (N->getValueType(0) != MVT::i32 && 12128 N->getValueType(0) != MVT::i64) 12129 return SDValue(); 12130 12131 if (!((N->getOperand(0).getValueType() == MVT::i1 && Subtarget.useCRBits()) || 12132 (N->getOperand(0).getValueType() == MVT::i32 && Subtarget.isPPC64()))) 12133 return SDValue(); 12134 12135 if (N->getOperand(0).getOpcode() != ISD::AND && 12136 N->getOperand(0).getOpcode() != ISD::OR && 12137 N->getOperand(0).getOpcode() != ISD::XOR && 12138 N->getOperand(0).getOpcode() != ISD::SELECT && 12139 N->getOperand(0).getOpcode() != ISD::SELECT_CC) 12140 return SDValue(); 12141 12142 SmallVector<SDValue, 4> Inputs; 12143 SmallVector<SDValue, 8> BinOps(1, N->getOperand(0)), PromOps; 12144 SmallPtrSet<SDNode *, 16> Visited; 12145 12146 // Visit all inputs, collect all binary operations (and, or, xor and 12147 // select) that are all fed by truncations. 12148 while (!BinOps.empty()) { 12149 SDValue BinOp = BinOps.back(); 12150 BinOps.pop_back(); 12151 12152 if (!Visited.insert(BinOp.getNode()).second) 12153 continue; 12154 12155 PromOps.push_back(BinOp); 12156 12157 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) { 12158 // The condition of the select is not promoted. 12159 if (BinOp.getOpcode() == ISD::SELECT && i == 0) 12160 continue; 12161 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3) 12162 continue; 12163 12164 if (BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE || 12165 isa<ConstantSDNode>(BinOp.getOperand(i))) { 12166 Inputs.push_back(BinOp.getOperand(i)); 12167 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND || 12168 BinOp.getOperand(i).getOpcode() == ISD::OR || 12169 BinOp.getOperand(i).getOpcode() == ISD::XOR || 12170 BinOp.getOperand(i).getOpcode() == ISD::SELECT || 12171 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC) { 12172 BinOps.push_back(BinOp.getOperand(i)); 12173 } else { 12174 // We have an input that is not a truncation or another binary 12175 // operation; we'll abort this transformation. 12176 return SDValue(); 12177 } 12178 } 12179 } 12180 12181 // The operands of a select that must be truncated when the select is 12182 // promoted because the operand is actually part of the to-be-promoted set. 12183 DenseMap<SDNode *, EVT> SelectTruncOp[2]; 12184 12185 // Make sure that this is a self-contained cluster of operations (which 12186 // is not quite the same thing as saying that everything has only one 12187 // use). 12188 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 12189 if (isa<ConstantSDNode>(Inputs[i])) 12190 continue; 12191 12192 for (SDNode::use_iterator UI = Inputs[i].getNode()->use_begin(), 12193 UE = Inputs[i].getNode()->use_end(); 12194 UI != UE; ++UI) { 12195 SDNode *User = *UI; 12196 if (User != N && !Visited.count(User)) 12197 return SDValue(); 12198 12199 // If we're going to promote the non-output-value operand(s) or SELECT or 12200 // SELECT_CC, record them for truncation. 12201 if (User->getOpcode() == ISD::SELECT) { 12202 if (User->getOperand(0) == Inputs[i]) 12203 SelectTruncOp[0].insert(std::make_pair(User, 12204 User->getOperand(0).getValueType())); 12205 } else if (User->getOpcode() == ISD::SELECT_CC) { 12206 if (User->getOperand(0) == Inputs[i]) 12207 SelectTruncOp[0].insert(std::make_pair(User, 12208 User->getOperand(0).getValueType())); 12209 if (User->getOperand(1) == Inputs[i]) 12210 SelectTruncOp[1].insert(std::make_pair(User, 12211 User->getOperand(1).getValueType())); 12212 } 12213 } 12214 } 12215 12216 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) { 12217 for (SDNode::use_iterator UI = PromOps[i].getNode()->use_begin(), 12218 UE = PromOps[i].getNode()->use_end(); 12219 UI != UE; ++UI) { 12220 SDNode *User = *UI; 12221 if (User != N && !Visited.count(User)) 12222 return SDValue(); 12223 12224 // If we're going to promote the non-output-value operand(s) or SELECT or 12225 // SELECT_CC, record them for truncation. 12226 if (User->getOpcode() == ISD::SELECT) { 12227 if (User->getOperand(0) == PromOps[i]) 12228 SelectTruncOp[0].insert(std::make_pair(User, 12229 User->getOperand(0).getValueType())); 12230 } else if (User->getOpcode() == ISD::SELECT_CC) { 12231 if (User->getOperand(0) == PromOps[i]) 12232 SelectTruncOp[0].insert(std::make_pair(User, 12233 User->getOperand(0).getValueType())); 12234 if (User->getOperand(1) == PromOps[i]) 12235 SelectTruncOp[1].insert(std::make_pair(User, 12236 User->getOperand(1).getValueType())); 12237 } 12238 } 12239 } 12240 12241 unsigned PromBits = N->getOperand(0).getValueSizeInBits(); 12242 bool ReallyNeedsExt = false; 12243 if (N->getOpcode() != ISD::ANY_EXTEND) { 12244 // If all of the inputs are not already sign/zero extended, then 12245 // we'll still need to do that at the end. 12246 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 12247 if (isa<ConstantSDNode>(Inputs[i])) 12248 continue; 12249 12250 unsigned OpBits = 12251 Inputs[i].getOperand(0).getValueSizeInBits(); 12252 assert(PromBits < OpBits && "Truncation not to a smaller bit count?"); 12253 12254 if ((N->getOpcode() == ISD::ZERO_EXTEND && 12255 !DAG.MaskedValueIsZero(Inputs[i].getOperand(0), 12256 APInt::getHighBitsSet(OpBits, 12257 OpBits-PromBits))) || 12258 (N->getOpcode() == ISD::SIGN_EXTEND && 12259 DAG.ComputeNumSignBits(Inputs[i].getOperand(0)) < 12260 (OpBits-(PromBits-1)))) { 12261 ReallyNeedsExt = true; 12262 break; 12263 } 12264 } 12265 } 12266 12267 // Replace all inputs, either with the truncation operand, or a 12268 // truncation or extension to the final output type. 12269 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 12270 // Constant inputs need to be replaced with the to-be-promoted nodes that 12271 // use them because they might have users outside of the cluster of 12272 // promoted nodes. 12273 if (isa<ConstantSDNode>(Inputs[i])) 12274 continue; 12275 12276 SDValue InSrc = Inputs[i].getOperand(0); 12277 if (Inputs[i].getValueType() == N->getValueType(0)) 12278 DAG.ReplaceAllUsesOfValueWith(Inputs[i], InSrc); 12279 else if (N->getOpcode() == ISD::SIGN_EXTEND) 12280 DAG.ReplaceAllUsesOfValueWith(Inputs[i], 12281 DAG.getSExtOrTrunc(InSrc, dl, N->getValueType(0))); 12282 else if (N->getOpcode() == ISD::ZERO_EXTEND) 12283 DAG.ReplaceAllUsesOfValueWith(Inputs[i], 12284 DAG.getZExtOrTrunc(InSrc, dl, N->getValueType(0))); 12285 else 12286 DAG.ReplaceAllUsesOfValueWith(Inputs[i], 12287 DAG.getAnyExtOrTrunc(InSrc, dl, N->getValueType(0))); 12288 } 12289 12290 std::list<HandleSDNode> PromOpHandles; 12291 for (auto &PromOp : PromOps) 12292 PromOpHandles.emplace_back(PromOp); 12293 12294 // Replace all operations (these are all the same, but have a different 12295 // (promoted) return type). DAG.getNode will validate that the types of 12296 // a binary operator match, so go through the list in reverse so that 12297 // we've likely promoted both operands first. 12298 while (!PromOpHandles.empty()) { 12299 SDValue PromOp = PromOpHandles.back().getValue(); 12300 PromOpHandles.pop_back(); 12301 12302 unsigned C; 12303 switch (PromOp.getOpcode()) { 12304 default: C = 0; break; 12305 case ISD::SELECT: C = 1; break; 12306 case ISD::SELECT_CC: C = 2; break; 12307 } 12308 12309 if ((!isa<ConstantSDNode>(PromOp.getOperand(C)) && 12310 PromOp.getOperand(C).getValueType() != N->getValueType(0)) || 12311 (!isa<ConstantSDNode>(PromOp.getOperand(C+1)) && 12312 PromOp.getOperand(C+1).getValueType() != N->getValueType(0))) { 12313 // The to-be-promoted operands of this node have not yet been 12314 // promoted (this should be rare because we're going through the 12315 // list backward, but if one of the operands has several users in 12316 // this cluster of to-be-promoted nodes, it is possible). 12317 PromOpHandles.emplace_front(PromOp); 12318 continue; 12319 } 12320 12321 // For SELECT and SELECT_CC nodes, we do a similar check for any 12322 // to-be-promoted comparison inputs. 12323 if (PromOp.getOpcode() == ISD::SELECT || 12324 PromOp.getOpcode() == ISD::SELECT_CC) { 12325 if ((SelectTruncOp[0].count(PromOp.getNode()) && 12326 PromOp.getOperand(0).getValueType() != N->getValueType(0)) || 12327 (SelectTruncOp[1].count(PromOp.getNode()) && 12328 PromOp.getOperand(1).getValueType() != N->getValueType(0))) { 12329 PromOpHandles.emplace_front(PromOp); 12330 continue; 12331 } 12332 } 12333 12334 SmallVector<SDValue, 3> Ops(PromOp.getNode()->op_begin(), 12335 PromOp.getNode()->op_end()); 12336 12337 // If this node has constant inputs, then they'll need to be promoted here. 12338 for (unsigned i = 0; i < 2; ++i) { 12339 if (!isa<ConstantSDNode>(Ops[C+i])) 12340 continue; 12341 if (Ops[C+i].getValueType() == N->getValueType(0)) 12342 continue; 12343 12344 if (N->getOpcode() == ISD::SIGN_EXTEND) 12345 Ops[C+i] = DAG.getSExtOrTrunc(Ops[C+i], dl, N->getValueType(0)); 12346 else if (N->getOpcode() == ISD::ZERO_EXTEND) 12347 Ops[C+i] = DAG.getZExtOrTrunc(Ops[C+i], dl, N->getValueType(0)); 12348 else 12349 Ops[C+i] = DAG.getAnyExtOrTrunc(Ops[C+i], dl, N->getValueType(0)); 12350 } 12351 12352 // If we've promoted the comparison inputs of a SELECT or SELECT_CC, 12353 // truncate them again to the original value type. 12354 if (PromOp.getOpcode() == ISD::SELECT || 12355 PromOp.getOpcode() == ISD::SELECT_CC) { 12356 auto SI0 = SelectTruncOp[0].find(PromOp.getNode()); 12357 if (SI0 != SelectTruncOp[0].end()) 12358 Ops[0] = DAG.getNode(ISD::TRUNCATE, dl, SI0->second, Ops[0]); 12359 auto SI1 = SelectTruncOp[1].find(PromOp.getNode()); 12360 if (SI1 != SelectTruncOp[1].end()) 12361 Ops[1] = DAG.getNode(ISD::TRUNCATE, dl, SI1->second, Ops[1]); 12362 } 12363 12364 DAG.ReplaceAllUsesOfValueWith(PromOp, 12365 DAG.getNode(PromOp.getOpcode(), dl, N->getValueType(0), Ops)); 12366 } 12367 12368 // Now we're left with the initial extension itself. 12369 if (!ReallyNeedsExt) 12370 return N->getOperand(0); 12371 12372 // To zero extend, just mask off everything except for the first bit (in the 12373 // i1 case). 12374 if (N->getOpcode() == ISD::ZERO_EXTEND) 12375 return DAG.getNode(ISD::AND, dl, N->getValueType(0), N->getOperand(0), 12376 DAG.getConstant(APInt::getLowBitsSet( 12377 N->getValueSizeInBits(0), PromBits), 12378 dl, N->getValueType(0))); 12379 12380 assert(N->getOpcode() == ISD::SIGN_EXTEND && 12381 "Invalid extension type"); 12382 EVT ShiftAmountTy = getShiftAmountTy(N->getValueType(0), DAG.getDataLayout()); 12383 SDValue ShiftCst = 12384 DAG.getConstant(N->getValueSizeInBits(0) - PromBits, dl, ShiftAmountTy); 12385 return DAG.getNode( 12386 ISD::SRA, dl, N->getValueType(0), 12387 DAG.getNode(ISD::SHL, dl, N->getValueType(0), N->getOperand(0), ShiftCst), 12388 ShiftCst); 12389 } 12390 12391 SDValue PPCTargetLowering::combineSetCC(SDNode *N, 12392 DAGCombinerInfo &DCI) const { 12393 assert(N->getOpcode() == ISD::SETCC && 12394 "Should be called with a SETCC node"); 12395 12396 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get(); 12397 if (CC == ISD::SETNE || CC == ISD::SETEQ) { 12398 SDValue LHS = N->getOperand(0); 12399 SDValue RHS = N->getOperand(1); 12400 12401 // If there is a '0 - y' pattern, canonicalize the pattern to the RHS. 12402 if (LHS.getOpcode() == ISD::SUB && isNullConstant(LHS.getOperand(0)) && 12403 LHS.hasOneUse()) 12404 std::swap(LHS, RHS); 12405 12406 // x == 0-y --> x+y == 0 12407 // x != 0-y --> x+y != 0 12408 if (RHS.getOpcode() == ISD::SUB && isNullConstant(RHS.getOperand(0)) && 12409 RHS.hasOneUse()) { 12410 SDLoc DL(N); 12411 SelectionDAG &DAG = DCI.DAG; 12412 EVT VT = N->getValueType(0); 12413 EVT OpVT = LHS.getValueType(); 12414 SDValue Add = DAG.getNode(ISD::ADD, DL, OpVT, LHS, RHS.getOperand(1)); 12415 return DAG.getSetCC(DL, VT, Add, DAG.getConstant(0, DL, OpVT), CC); 12416 } 12417 } 12418 12419 return DAGCombineTruncBoolExt(N, DCI); 12420 } 12421 12422 // Is this an extending load from an f32 to an f64? 12423 static bool isFPExtLoad(SDValue Op) { 12424 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op.getNode())) 12425 return LD->getExtensionType() == ISD::EXTLOAD && 12426 Op.getValueType() == MVT::f64; 12427 return false; 12428 } 12429 12430 /// Reduces the number of fp-to-int conversion when building a vector. 12431 /// 12432 /// If this vector is built out of floating to integer conversions, 12433 /// transform it to a vector built out of floating point values followed by a 12434 /// single floating to integer conversion of the vector. 12435 /// Namely (build_vector (fptosi $A), (fptosi $B), ...) 12436 /// becomes (fptosi (build_vector ($A, $B, ...))) 12437 SDValue PPCTargetLowering:: 12438 combineElementTruncationToVectorTruncation(SDNode *N, 12439 DAGCombinerInfo &DCI) const { 12440 assert(N->getOpcode() == ISD::BUILD_VECTOR && 12441 "Should be called with a BUILD_VECTOR node"); 12442 12443 SelectionDAG &DAG = DCI.DAG; 12444 SDLoc dl(N); 12445 12446 SDValue FirstInput = N->getOperand(0); 12447 assert(FirstInput.getOpcode() == PPCISD::MFVSR && 12448 "The input operand must be an fp-to-int conversion."); 12449 12450 // This combine happens after legalization so the fp_to_[su]i nodes are 12451 // already converted to PPCSISD nodes. 12452 unsigned FirstConversion = FirstInput.getOperand(0).getOpcode(); 12453 if (FirstConversion == PPCISD::FCTIDZ || 12454 FirstConversion == PPCISD::FCTIDUZ || 12455 FirstConversion == PPCISD::FCTIWZ || 12456 FirstConversion == PPCISD::FCTIWUZ) { 12457 bool IsSplat = true; 12458 bool Is32Bit = FirstConversion == PPCISD::FCTIWZ || 12459 FirstConversion == PPCISD::FCTIWUZ; 12460 EVT SrcVT = FirstInput.getOperand(0).getValueType(); 12461 SmallVector<SDValue, 4> Ops; 12462 EVT TargetVT = N->getValueType(0); 12463 for (int i = 0, e = N->getNumOperands(); i < e; ++i) { 12464 SDValue NextOp = N->getOperand(i); 12465 if (NextOp.getOpcode() != PPCISD::MFVSR) 12466 return SDValue(); 12467 unsigned NextConversion = NextOp.getOperand(0).getOpcode(); 12468 if (NextConversion != FirstConversion) 12469 return SDValue(); 12470 // If we are converting to 32-bit integers, we need to add an FP_ROUND. 12471 // This is not valid if the input was originally double precision. It is 12472 // also not profitable to do unless this is an extending load in which 12473 // case doing this combine will allow us to combine consecutive loads. 12474 if (Is32Bit && !isFPExtLoad(NextOp.getOperand(0).getOperand(0))) 12475 return SDValue(); 12476 if (N->getOperand(i) != FirstInput) 12477 IsSplat = false; 12478 } 12479 12480 // If this is a splat, we leave it as-is since there will be only a single 12481 // fp-to-int conversion followed by a splat of the integer. This is better 12482 // for 32-bit and smaller ints and neutral for 64-bit ints. 12483 if (IsSplat) 12484 return SDValue(); 12485 12486 // Now that we know we have the right type of node, get its operands 12487 for (int i = 0, e = N->getNumOperands(); i < e; ++i) { 12488 SDValue In = N->getOperand(i).getOperand(0); 12489 if (Is32Bit) { 12490 // For 32-bit values, we need to add an FP_ROUND node (if we made it 12491 // here, we know that all inputs are extending loads so this is safe). 12492 if (In.isUndef()) 12493 Ops.push_back(DAG.getUNDEF(SrcVT)); 12494 else { 12495 SDValue Trunc = DAG.getNode(ISD::FP_ROUND, dl, 12496 MVT::f32, In.getOperand(0), 12497 DAG.getIntPtrConstant(1, dl)); 12498 Ops.push_back(Trunc); 12499 } 12500 } else 12501 Ops.push_back(In.isUndef() ? DAG.getUNDEF(SrcVT) : In.getOperand(0)); 12502 } 12503 12504 unsigned Opcode; 12505 if (FirstConversion == PPCISD::FCTIDZ || 12506 FirstConversion == PPCISD::FCTIWZ) 12507 Opcode = ISD::FP_TO_SINT; 12508 else 12509 Opcode = ISD::FP_TO_UINT; 12510 12511 EVT NewVT = TargetVT == MVT::v2i64 ? MVT::v2f64 : MVT::v4f32; 12512 SDValue BV = DAG.getBuildVector(NewVT, dl, Ops); 12513 return DAG.getNode(Opcode, dl, TargetVT, BV); 12514 } 12515 return SDValue(); 12516 } 12517 12518 /// Reduce the number of loads when building a vector. 12519 /// 12520 /// Building a vector out of multiple loads can be converted to a load 12521 /// of the vector type if the loads are consecutive. If the loads are 12522 /// consecutive but in descending order, a shuffle is added at the end 12523 /// to reorder the vector. 12524 static SDValue combineBVOfConsecutiveLoads(SDNode *N, SelectionDAG &DAG) { 12525 assert(N->getOpcode() == ISD::BUILD_VECTOR && 12526 "Should be called with a BUILD_VECTOR node"); 12527 12528 SDLoc dl(N); 12529 12530 // Return early for non byte-sized type, as they can't be consecutive. 12531 if (!N->getValueType(0).getVectorElementType().isByteSized()) 12532 return SDValue(); 12533 12534 bool InputsAreConsecutiveLoads = true; 12535 bool InputsAreReverseConsecutive = true; 12536 unsigned ElemSize = N->getValueType(0).getScalarType().getStoreSize(); 12537 SDValue FirstInput = N->getOperand(0); 12538 bool IsRoundOfExtLoad = false; 12539 12540 if (FirstInput.getOpcode() == ISD::FP_ROUND && 12541 FirstInput.getOperand(0).getOpcode() == ISD::LOAD) { 12542 LoadSDNode *LD = dyn_cast<LoadSDNode>(FirstInput.getOperand(0)); 12543 IsRoundOfExtLoad = LD->getExtensionType() == ISD::EXTLOAD; 12544 } 12545 // Not a build vector of (possibly fp_rounded) loads. 12546 if ((!IsRoundOfExtLoad && FirstInput.getOpcode() != ISD::LOAD) || 12547 N->getNumOperands() == 1) 12548 return SDValue(); 12549 12550 for (int i = 1, e = N->getNumOperands(); i < e; ++i) { 12551 // If any inputs are fp_round(extload), they all must be. 12552 if (IsRoundOfExtLoad && N->getOperand(i).getOpcode() != ISD::FP_ROUND) 12553 return SDValue(); 12554 12555 SDValue NextInput = IsRoundOfExtLoad ? N->getOperand(i).getOperand(0) : 12556 N->getOperand(i); 12557 if (NextInput.getOpcode() != ISD::LOAD) 12558 return SDValue(); 12559 12560 SDValue PreviousInput = 12561 IsRoundOfExtLoad ? N->getOperand(i-1).getOperand(0) : N->getOperand(i-1); 12562 LoadSDNode *LD1 = dyn_cast<LoadSDNode>(PreviousInput); 12563 LoadSDNode *LD2 = dyn_cast<LoadSDNode>(NextInput); 12564 12565 // If any inputs are fp_round(extload), they all must be. 12566 if (IsRoundOfExtLoad && LD2->getExtensionType() != ISD::EXTLOAD) 12567 return SDValue(); 12568 12569 if (!isConsecutiveLS(LD2, LD1, ElemSize, 1, DAG)) 12570 InputsAreConsecutiveLoads = false; 12571 if (!isConsecutiveLS(LD1, LD2, ElemSize, 1, DAG)) 12572 InputsAreReverseConsecutive = false; 12573 12574 // Exit early if the loads are neither consecutive nor reverse consecutive. 12575 if (!InputsAreConsecutiveLoads && !InputsAreReverseConsecutive) 12576 return SDValue(); 12577 } 12578 12579 assert(!(InputsAreConsecutiveLoads && InputsAreReverseConsecutive) && 12580 "The loads cannot be both consecutive and reverse consecutive."); 12581 12582 SDValue FirstLoadOp = 12583 IsRoundOfExtLoad ? FirstInput.getOperand(0) : FirstInput; 12584 SDValue LastLoadOp = 12585 IsRoundOfExtLoad ? N->getOperand(N->getNumOperands()-1).getOperand(0) : 12586 N->getOperand(N->getNumOperands()-1); 12587 12588 LoadSDNode *LD1 = dyn_cast<LoadSDNode>(FirstLoadOp); 12589 LoadSDNode *LDL = dyn_cast<LoadSDNode>(LastLoadOp); 12590 if (InputsAreConsecutiveLoads) { 12591 assert(LD1 && "Input needs to be a LoadSDNode."); 12592 return DAG.getLoad(N->getValueType(0), dl, LD1->getChain(), 12593 LD1->getBasePtr(), LD1->getPointerInfo(), 12594 LD1->getAlignment()); 12595 } 12596 if (InputsAreReverseConsecutive) { 12597 assert(LDL && "Input needs to be a LoadSDNode."); 12598 SDValue Load = DAG.getLoad(N->getValueType(0), dl, LDL->getChain(), 12599 LDL->getBasePtr(), LDL->getPointerInfo(), 12600 LDL->getAlignment()); 12601 SmallVector<int, 16> Ops; 12602 for (int i = N->getNumOperands() - 1; i >= 0; i--) 12603 Ops.push_back(i); 12604 12605 return DAG.getVectorShuffle(N->getValueType(0), dl, Load, 12606 DAG.getUNDEF(N->getValueType(0)), Ops); 12607 } 12608 return SDValue(); 12609 } 12610 12611 // This function adds the required vector_shuffle needed to get 12612 // the elements of the vector extract in the correct position 12613 // as specified by the CorrectElems encoding. 12614 static SDValue addShuffleForVecExtend(SDNode *N, SelectionDAG &DAG, 12615 SDValue Input, uint64_t Elems, 12616 uint64_t CorrectElems) { 12617 SDLoc dl(N); 12618 12619 unsigned NumElems = Input.getValueType().getVectorNumElements(); 12620 SmallVector<int, 16> ShuffleMask(NumElems, -1); 12621 12622 // Knowing the element indices being extracted from the original 12623 // vector and the order in which they're being inserted, just put 12624 // them at element indices required for the instruction. 12625 for (unsigned i = 0; i < N->getNumOperands(); i++) { 12626 if (DAG.getDataLayout().isLittleEndian()) 12627 ShuffleMask[CorrectElems & 0xF] = Elems & 0xF; 12628 else 12629 ShuffleMask[(CorrectElems & 0xF0) >> 4] = (Elems & 0xF0) >> 4; 12630 CorrectElems = CorrectElems >> 8; 12631 Elems = Elems >> 8; 12632 } 12633 12634 SDValue Shuffle = 12635 DAG.getVectorShuffle(Input.getValueType(), dl, Input, 12636 DAG.getUNDEF(Input.getValueType()), ShuffleMask); 12637 12638 EVT Ty = N->getValueType(0); 12639 SDValue BV = DAG.getNode(PPCISD::SExtVElems, dl, Ty, Shuffle); 12640 return BV; 12641 } 12642 12643 // Look for build vector patterns where input operands come from sign 12644 // extended vector_extract elements of specific indices. If the correct indices 12645 // aren't used, add a vector shuffle to fix up the indices and create a new 12646 // PPCISD:SExtVElems node which selects the vector sign extend instructions 12647 // during instruction selection. 12648 static SDValue combineBVOfVecSExt(SDNode *N, SelectionDAG &DAG) { 12649 // This array encodes the indices that the vector sign extend instructions 12650 // extract from when extending from one type to another for both BE and LE. 12651 // The right nibble of each byte corresponds to the LE incides. 12652 // and the left nibble of each byte corresponds to the BE incides. 12653 // For example: 0x3074B8FC byte->word 12654 // For LE: the allowed indices are: 0x0,0x4,0x8,0xC 12655 // For BE: the allowed indices are: 0x3,0x7,0xB,0xF 12656 // For example: 0x000070F8 byte->double word 12657 // For LE: the allowed indices are: 0x0,0x8 12658 // For BE: the allowed indices are: 0x7,0xF 12659 uint64_t TargetElems[] = { 12660 0x3074B8FC, // b->w 12661 0x000070F8, // b->d 12662 0x10325476, // h->w 12663 0x00003074, // h->d 12664 0x00001032, // w->d 12665 }; 12666 12667 uint64_t Elems = 0; 12668 int Index; 12669 SDValue Input; 12670 12671 auto isSExtOfVecExtract = [&](SDValue Op) -> bool { 12672 if (!Op) 12673 return false; 12674 if (Op.getOpcode() != ISD::SIGN_EXTEND && 12675 Op.getOpcode() != ISD::SIGN_EXTEND_INREG) 12676 return false; 12677 12678 // A SIGN_EXTEND_INREG might be fed by an ANY_EXTEND to produce a value 12679 // of the right width. 12680 SDValue Extract = Op.getOperand(0); 12681 if (Extract.getOpcode() == ISD::ANY_EXTEND) 12682 Extract = Extract.getOperand(0); 12683 if (Extract.getOpcode() != ISD::EXTRACT_VECTOR_ELT) 12684 return false; 12685 12686 ConstantSDNode *ExtOp = dyn_cast<ConstantSDNode>(Extract.getOperand(1)); 12687 if (!ExtOp) 12688 return false; 12689 12690 Index = ExtOp->getZExtValue(); 12691 if (Input && Input != Extract.getOperand(0)) 12692 return false; 12693 12694 if (!Input) 12695 Input = Extract.getOperand(0); 12696 12697 Elems = Elems << 8; 12698 Index = DAG.getDataLayout().isLittleEndian() ? Index : Index << 4; 12699 Elems |= Index; 12700 12701 return true; 12702 }; 12703 12704 // If the build vector operands aren't sign extended vector extracts, 12705 // of the same input vector, then return. 12706 for (unsigned i = 0; i < N->getNumOperands(); i++) { 12707 if (!isSExtOfVecExtract(N->getOperand(i))) { 12708 return SDValue(); 12709 } 12710 } 12711 12712 // If the vector extract indicies are not correct, add the appropriate 12713 // vector_shuffle. 12714 int TgtElemArrayIdx; 12715 int InputSize = Input.getValueType().getScalarSizeInBits(); 12716 int OutputSize = N->getValueType(0).getScalarSizeInBits(); 12717 if (InputSize + OutputSize == 40) 12718 TgtElemArrayIdx = 0; 12719 else if (InputSize + OutputSize == 72) 12720 TgtElemArrayIdx = 1; 12721 else if (InputSize + OutputSize == 48) 12722 TgtElemArrayIdx = 2; 12723 else if (InputSize + OutputSize == 80) 12724 TgtElemArrayIdx = 3; 12725 else if (InputSize + OutputSize == 96) 12726 TgtElemArrayIdx = 4; 12727 else 12728 return SDValue(); 12729 12730 uint64_t CorrectElems = TargetElems[TgtElemArrayIdx]; 12731 CorrectElems = DAG.getDataLayout().isLittleEndian() 12732 ? CorrectElems & 0x0F0F0F0F0F0F0F0F 12733 : CorrectElems & 0xF0F0F0F0F0F0F0F0; 12734 if (Elems != CorrectElems) { 12735 return addShuffleForVecExtend(N, DAG, Input, Elems, CorrectElems); 12736 } 12737 12738 // Regular lowering will catch cases where a shuffle is not needed. 12739 return SDValue(); 12740 } 12741 12742 SDValue PPCTargetLowering::DAGCombineBuildVector(SDNode *N, 12743 DAGCombinerInfo &DCI) const { 12744 assert(N->getOpcode() == ISD::BUILD_VECTOR && 12745 "Should be called with a BUILD_VECTOR node"); 12746 12747 SelectionDAG &DAG = DCI.DAG; 12748 SDLoc dl(N); 12749 12750 if (!Subtarget.hasVSX()) 12751 return SDValue(); 12752 12753 // The target independent DAG combiner will leave a build_vector of 12754 // float-to-int conversions intact. We can generate MUCH better code for 12755 // a float-to-int conversion of a vector of floats. 12756 SDValue FirstInput = N->getOperand(0); 12757 if (FirstInput.getOpcode() == PPCISD::MFVSR) { 12758 SDValue Reduced = combineElementTruncationToVectorTruncation(N, DCI); 12759 if (Reduced) 12760 return Reduced; 12761 } 12762 12763 // If we're building a vector out of consecutive loads, just load that 12764 // vector type. 12765 SDValue Reduced = combineBVOfConsecutiveLoads(N, DAG); 12766 if (Reduced) 12767 return Reduced; 12768 12769 // If we're building a vector out of extended elements from another vector 12770 // we have P9 vector integer extend instructions. The code assumes legal 12771 // input types (i.e. it can't handle things like v4i16) so do not run before 12772 // legalization. 12773 if (Subtarget.hasP9Altivec() && !DCI.isBeforeLegalize()) { 12774 Reduced = combineBVOfVecSExt(N, DAG); 12775 if (Reduced) 12776 return Reduced; 12777 } 12778 12779 12780 if (N->getValueType(0) != MVT::v2f64) 12781 return SDValue(); 12782 12783 // Looking for: 12784 // (build_vector ([su]int_to_fp (extractelt 0)), [su]int_to_fp (extractelt 1)) 12785 if (FirstInput.getOpcode() != ISD::SINT_TO_FP && 12786 FirstInput.getOpcode() != ISD::UINT_TO_FP) 12787 return SDValue(); 12788 if (N->getOperand(1).getOpcode() != ISD::SINT_TO_FP && 12789 N->getOperand(1).getOpcode() != ISD::UINT_TO_FP) 12790 return SDValue(); 12791 if (FirstInput.getOpcode() != N->getOperand(1).getOpcode()) 12792 return SDValue(); 12793 12794 SDValue Ext1 = FirstInput.getOperand(0); 12795 SDValue Ext2 = N->getOperand(1).getOperand(0); 12796 if(Ext1.getOpcode() != ISD::EXTRACT_VECTOR_ELT || 12797 Ext2.getOpcode() != ISD::EXTRACT_VECTOR_ELT) 12798 return SDValue(); 12799 12800 ConstantSDNode *Ext1Op = dyn_cast<ConstantSDNode>(Ext1.getOperand(1)); 12801 ConstantSDNode *Ext2Op = dyn_cast<ConstantSDNode>(Ext2.getOperand(1)); 12802 if (!Ext1Op || !Ext2Op) 12803 return SDValue(); 12804 if (Ext1.getOperand(0).getValueType() != MVT::v4i32 || 12805 Ext1.getOperand(0) != Ext2.getOperand(0)) 12806 return SDValue(); 12807 12808 int FirstElem = Ext1Op->getZExtValue(); 12809 int SecondElem = Ext2Op->getZExtValue(); 12810 int SubvecIdx; 12811 if (FirstElem == 0 && SecondElem == 1) 12812 SubvecIdx = Subtarget.isLittleEndian() ? 1 : 0; 12813 else if (FirstElem == 2 && SecondElem == 3) 12814 SubvecIdx = Subtarget.isLittleEndian() ? 0 : 1; 12815 else 12816 return SDValue(); 12817 12818 SDValue SrcVec = Ext1.getOperand(0); 12819 auto NodeType = (N->getOperand(1).getOpcode() == ISD::SINT_TO_FP) ? 12820 PPCISD::SINT_VEC_TO_FP : PPCISD::UINT_VEC_TO_FP; 12821 return DAG.getNode(NodeType, dl, MVT::v2f64, 12822 SrcVec, DAG.getIntPtrConstant(SubvecIdx, dl)); 12823 } 12824 12825 SDValue PPCTargetLowering::combineFPToIntToFP(SDNode *N, 12826 DAGCombinerInfo &DCI) const { 12827 assert((N->getOpcode() == ISD::SINT_TO_FP || 12828 N->getOpcode() == ISD::UINT_TO_FP) && 12829 "Need an int -> FP conversion node here"); 12830 12831 if (useSoftFloat() || !Subtarget.has64BitSupport()) 12832 return SDValue(); 12833 12834 SelectionDAG &DAG = DCI.DAG; 12835 SDLoc dl(N); 12836 SDValue Op(N, 0); 12837 12838 // Don't handle ppc_fp128 here or conversions that are out-of-range capable 12839 // from the hardware. 12840 if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64) 12841 return SDValue(); 12842 if (Op.getOperand(0).getValueType().getSimpleVT() <= MVT(MVT::i1) || 12843 Op.getOperand(0).getValueType().getSimpleVT() > MVT(MVT::i64)) 12844 return SDValue(); 12845 12846 SDValue FirstOperand(Op.getOperand(0)); 12847 bool SubWordLoad = FirstOperand.getOpcode() == ISD::LOAD && 12848 (FirstOperand.getValueType() == MVT::i8 || 12849 FirstOperand.getValueType() == MVT::i16); 12850 if (Subtarget.hasP9Vector() && Subtarget.hasP9Altivec() && SubWordLoad) { 12851 bool Signed = N->getOpcode() == ISD::SINT_TO_FP; 12852 bool DstDouble = Op.getValueType() == MVT::f64; 12853 unsigned ConvOp = Signed ? 12854 (DstDouble ? PPCISD::FCFID : PPCISD::FCFIDS) : 12855 (DstDouble ? PPCISD::FCFIDU : PPCISD::FCFIDUS); 12856 SDValue WidthConst = 12857 DAG.getIntPtrConstant(FirstOperand.getValueType() == MVT::i8 ? 1 : 2, 12858 dl, false); 12859 LoadSDNode *LDN = cast<LoadSDNode>(FirstOperand.getNode()); 12860 SDValue Ops[] = { LDN->getChain(), LDN->getBasePtr(), WidthConst }; 12861 SDValue Ld = DAG.getMemIntrinsicNode(PPCISD::LXSIZX, dl, 12862 DAG.getVTList(MVT::f64, MVT::Other), 12863 Ops, MVT::i8, LDN->getMemOperand()); 12864 12865 // For signed conversion, we need to sign-extend the value in the VSR 12866 if (Signed) { 12867 SDValue ExtOps[] = { Ld, WidthConst }; 12868 SDValue Ext = DAG.getNode(PPCISD::VEXTS, dl, MVT::f64, ExtOps); 12869 return DAG.getNode(ConvOp, dl, DstDouble ? MVT::f64 : MVT::f32, Ext); 12870 } else 12871 return DAG.getNode(ConvOp, dl, DstDouble ? MVT::f64 : MVT::f32, Ld); 12872 } 12873 12874 12875 // For i32 intermediate values, unfortunately, the conversion functions 12876 // leave the upper 32 bits of the value are undefined. Within the set of 12877 // scalar instructions, we have no method for zero- or sign-extending the 12878 // value. Thus, we cannot handle i32 intermediate values here. 12879 if (Op.getOperand(0).getValueType() == MVT::i32) 12880 return SDValue(); 12881 12882 assert((Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) && 12883 "UINT_TO_FP is supported only with FPCVT"); 12884 12885 // If we have FCFIDS, then use it when converting to single-precision. 12886 // Otherwise, convert to double-precision and then round. 12887 unsigned FCFOp = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) 12888 ? (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDUS 12889 : PPCISD::FCFIDS) 12890 : (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDU 12891 : PPCISD::FCFID); 12892 MVT FCFTy = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) 12893 ? MVT::f32 12894 : MVT::f64; 12895 12896 // If we're converting from a float, to an int, and back to a float again, 12897 // then we don't need the store/load pair at all. 12898 if ((Op.getOperand(0).getOpcode() == ISD::FP_TO_UINT && 12899 Subtarget.hasFPCVT()) || 12900 (Op.getOperand(0).getOpcode() == ISD::FP_TO_SINT)) { 12901 SDValue Src = Op.getOperand(0).getOperand(0); 12902 if (Src.getValueType() == MVT::f32) { 12903 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src); 12904 DCI.AddToWorklist(Src.getNode()); 12905 } else if (Src.getValueType() != MVT::f64) { 12906 // Make sure that we don't pick up a ppc_fp128 source value. 12907 return SDValue(); 12908 } 12909 12910 unsigned FCTOp = 12911 Op.getOperand(0).getOpcode() == ISD::FP_TO_SINT ? PPCISD::FCTIDZ : 12912 PPCISD::FCTIDUZ; 12913 12914 SDValue Tmp = DAG.getNode(FCTOp, dl, MVT::f64, Src); 12915 SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Tmp); 12916 12917 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) { 12918 FP = DAG.getNode(ISD::FP_ROUND, dl, 12919 MVT::f32, FP, DAG.getIntPtrConstant(0, dl)); 12920 DCI.AddToWorklist(FP.getNode()); 12921 } 12922 12923 return FP; 12924 } 12925 12926 return SDValue(); 12927 } 12928 12929 // expandVSXLoadForLE - Convert VSX loads (which may be intrinsics for 12930 // builtins) into loads with swaps. 12931 SDValue PPCTargetLowering::expandVSXLoadForLE(SDNode *N, 12932 DAGCombinerInfo &DCI) const { 12933 SelectionDAG &DAG = DCI.DAG; 12934 SDLoc dl(N); 12935 SDValue Chain; 12936 SDValue Base; 12937 MachineMemOperand *MMO; 12938 12939 switch (N->getOpcode()) { 12940 default: 12941 llvm_unreachable("Unexpected opcode for little endian VSX load"); 12942 case ISD::LOAD: { 12943 LoadSDNode *LD = cast<LoadSDNode>(N); 12944 Chain = LD->getChain(); 12945 Base = LD->getBasePtr(); 12946 MMO = LD->getMemOperand(); 12947 // If the MMO suggests this isn't a load of a full vector, leave 12948 // things alone. For a built-in, we have to make the change for 12949 // correctness, so if there is a size problem that will be a bug. 12950 if (MMO->getSize() < 16) 12951 return SDValue(); 12952 break; 12953 } 12954 case ISD::INTRINSIC_W_CHAIN: { 12955 MemIntrinsicSDNode *Intrin = cast<MemIntrinsicSDNode>(N); 12956 Chain = Intrin->getChain(); 12957 // Similarly to the store case below, Intrin->getBasePtr() doesn't get 12958 // us what we want. Get operand 2 instead. 12959 Base = Intrin->getOperand(2); 12960 MMO = Intrin->getMemOperand(); 12961 break; 12962 } 12963 } 12964 12965 MVT VecTy = N->getValueType(0).getSimpleVT(); 12966 12967 // Do not expand to PPCISD::LXVD2X + PPCISD::XXSWAPD when the load is 12968 // aligned and the type is a vector with elements up to 4 bytes 12969 if (Subtarget.needsSwapsForVSXMemOps() && !(MMO->getAlignment()%16) 12970 && VecTy.getScalarSizeInBits() <= 32 ) { 12971 return SDValue(); 12972 } 12973 12974 SDValue LoadOps[] = { Chain, Base }; 12975 SDValue Load = DAG.getMemIntrinsicNode(PPCISD::LXVD2X, dl, 12976 DAG.getVTList(MVT::v2f64, MVT::Other), 12977 LoadOps, MVT::v2f64, MMO); 12978 12979 DCI.AddToWorklist(Load.getNode()); 12980 Chain = Load.getValue(1); 12981 SDValue Swap = DAG.getNode( 12982 PPCISD::XXSWAPD, dl, DAG.getVTList(MVT::v2f64, MVT::Other), Chain, Load); 12983 DCI.AddToWorklist(Swap.getNode()); 12984 12985 // Add a bitcast if the resulting load type doesn't match v2f64. 12986 if (VecTy != MVT::v2f64) { 12987 SDValue N = DAG.getNode(ISD::BITCAST, dl, VecTy, Swap); 12988 DCI.AddToWorklist(N.getNode()); 12989 // Package {bitcast value, swap's chain} to match Load's shape. 12990 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(VecTy, MVT::Other), 12991 N, Swap.getValue(1)); 12992 } 12993 12994 return Swap; 12995 } 12996 12997 // expandVSXStoreForLE - Convert VSX stores (which may be intrinsics for 12998 // builtins) into stores with swaps. 12999 SDValue PPCTargetLowering::expandVSXStoreForLE(SDNode *N, 13000 DAGCombinerInfo &DCI) const { 13001 SelectionDAG &DAG = DCI.DAG; 13002 SDLoc dl(N); 13003 SDValue Chain; 13004 SDValue Base; 13005 unsigned SrcOpnd; 13006 MachineMemOperand *MMO; 13007 13008 switch (N->getOpcode()) { 13009 default: 13010 llvm_unreachable("Unexpected opcode for little endian VSX store"); 13011 case ISD::STORE: { 13012 StoreSDNode *ST = cast<StoreSDNode>(N); 13013 Chain = ST->getChain(); 13014 Base = ST->getBasePtr(); 13015 MMO = ST->getMemOperand(); 13016 SrcOpnd = 1; 13017 // If the MMO suggests this isn't a store of a full vector, leave 13018 // things alone. For a built-in, we have to make the change for 13019 // correctness, so if there is a size problem that will be a bug. 13020 if (MMO->getSize() < 16) 13021 return SDValue(); 13022 break; 13023 } 13024 case ISD::INTRINSIC_VOID: { 13025 MemIntrinsicSDNode *Intrin = cast<MemIntrinsicSDNode>(N); 13026 Chain = Intrin->getChain(); 13027 // Intrin->getBasePtr() oddly does not get what we want. 13028 Base = Intrin->getOperand(3); 13029 MMO = Intrin->getMemOperand(); 13030 SrcOpnd = 2; 13031 break; 13032 } 13033 } 13034 13035 SDValue Src = N->getOperand(SrcOpnd); 13036 MVT VecTy = Src.getValueType().getSimpleVT(); 13037 13038 // Do not expand to PPCISD::XXSWAPD and PPCISD::STXVD2X when the load is 13039 // aligned and the type is a vector with elements up to 4 bytes 13040 if (Subtarget.needsSwapsForVSXMemOps() && !(MMO->getAlignment()%16) 13041 && VecTy.getScalarSizeInBits() <= 32 ) { 13042 return SDValue(); 13043 } 13044 13045 // All stores are done as v2f64 and possible bit cast. 13046 if (VecTy != MVT::v2f64) { 13047 Src = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Src); 13048 DCI.AddToWorklist(Src.getNode()); 13049 } 13050 13051 SDValue Swap = DAG.getNode(PPCISD::XXSWAPD, dl, 13052 DAG.getVTList(MVT::v2f64, MVT::Other), Chain, Src); 13053 DCI.AddToWorklist(Swap.getNode()); 13054 Chain = Swap.getValue(1); 13055 SDValue StoreOps[] = { Chain, Swap, Base }; 13056 SDValue Store = DAG.getMemIntrinsicNode(PPCISD::STXVD2X, dl, 13057 DAG.getVTList(MVT::Other), 13058 StoreOps, VecTy, MMO); 13059 DCI.AddToWorklist(Store.getNode()); 13060 return Store; 13061 } 13062 13063 // Handle DAG combine for STORE (FP_TO_INT F). 13064 SDValue PPCTargetLowering::combineStoreFPToInt(SDNode *N, 13065 DAGCombinerInfo &DCI) const { 13066 13067 SelectionDAG &DAG = DCI.DAG; 13068 SDLoc dl(N); 13069 unsigned Opcode = N->getOperand(1).getOpcode(); 13070 13071 assert((Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT) 13072 && "Not a FP_TO_INT Instruction!"); 13073 13074 SDValue Val = N->getOperand(1).getOperand(0); 13075 EVT Op1VT = N->getOperand(1).getValueType(); 13076 EVT ResVT = Val.getValueType(); 13077 13078 // Floating point types smaller than 32 bits are not legal on Power. 13079 if (ResVT.getScalarSizeInBits() < 32) 13080 return SDValue(); 13081 13082 // Only perform combine for conversion to i64/i32 or power9 i16/i8. 13083 bool ValidTypeForStoreFltAsInt = 13084 (Op1VT == MVT::i32 || Op1VT == MVT::i64 || 13085 (Subtarget.hasP9Vector() && (Op1VT == MVT::i16 || Op1VT == MVT::i8))); 13086 13087 if (ResVT == MVT::ppcf128 || !Subtarget.hasP8Altivec() || 13088 cast<StoreSDNode>(N)->isTruncatingStore() || !ValidTypeForStoreFltAsInt) 13089 return SDValue(); 13090 13091 // Extend f32 values to f64 13092 if (ResVT.getScalarSizeInBits() == 32) { 13093 Val = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Val); 13094 DCI.AddToWorklist(Val.getNode()); 13095 } 13096 13097 // Set signed or unsigned conversion opcode. 13098 unsigned ConvOpcode = (Opcode == ISD::FP_TO_SINT) ? 13099 PPCISD::FP_TO_SINT_IN_VSR : 13100 PPCISD::FP_TO_UINT_IN_VSR; 13101 13102 Val = DAG.getNode(ConvOpcode, 13103 dl, ResVT == MVT::f128 ? MVT::f128 : MVT::f64, Val); 13104 DCI.AddToWorklist(Val.getNode()); 13105 13106 // Set number of bytes being converted. 13107 unsigned ByteSize = Op1VT.getScalarSizeInBits() / 8; 13108 SDValue Ops[] = { N->getOperand(0), Val, N->getOperand(2), 13109 DAG.getIntPtrConstant(ByteSize, dl, false), 13110 DAG.getValueType(Op1VT) }; 13111 13112 Val = DAG.getMemIntrinsicNode(PPCISD::ST_VSR_SCAL_INT, dl, 13113 DAG.getVTList(MVT::Other), Ops, 13114 cast<StoreSDNode>(N)->getMemoryVT(), 13115 cast<StoreSDNode>(N)->getMemOperand()); 13116 13117 DCI.AddToWorklist(Val.getNode()); 13118 return Val; 13119 } 13120 13121 SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N, 13122 DAGCombinerInfo &DCI) const { 13123 SelectionDAG &DAG = DCI.DAG; 13124 SDLoc dl(N); 13125 switch (N->getOpcode()) { 13126 default: break; 13127 case ISD::ADD: 13128 return combineADD(N, DCI); 13129 case ISD::SHL: 13130 return combineSHL(N, DCI); 13131 case ISD::SRA: 13132 return combineSRA(N, DCI); 13133 case ISD::SRL: 13134 return combineSRL(N, DCI); 13135 case ISD::MUL: 13136 return combineMUL(N, DCI); 13137 case PPCISD::SHL: 13138 if (isNullConstant(N->getOperand(0))) // 0 << V -> 0. 13139 return N->getOperand(0); 13140 break; 13141 case PPCISD::SRL: 13142 if (isNullConstant(N->getOperand(0))) // 0 >>u V -> 0. 13143 return N->getOperand(0); 13144 break; 13145 case PPCISD::SRA: 13146 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(0))) { 13147 if (C->isNullValue() || // 0 >>s V -> 0. 13148 C->isAllOnesValue()) // -1 >>s V -> -1. 13149 return N->getOperand(0); 13150 } 13151 break; 13152 case ISD::SIGN_EXTEND: 13153 case ISD::ZERO_EXTEND: 13154 case ISD::ANY_EXTEND: 13155 return DAGCombineExtBoolTrunc(N, DCI); 13156 case ISD::TRUNCATE: 13157 return combineTRUNCATE(N, DCI); 13158 case ISD::SETCC: 13159 if (SDValue CSCC = combineSetCC(N, DCI)) 13160 return CSCC; 13161 LLVM_FALLTHROUGH; 13162 case ISD::SELECT_CC: 13163 return DAGCombineTruncBoolExt(N, DCI); 13164 case ISD::SINT_TO_FP: 13165 case ISD::UINT_TO_FP: 13166 return combineFPToIntToFP(N, DCI); 13167 case ISD::STORE: { 13168 13169 EVT Op1VT = N->getOperand(1).getValueType(); 13170 unsigned Opcode = N->getOperand(1).getOpcode(); 13171 13172 if (Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT) { 13173 SDValue Val= combineStoreFPToInt(N, DCI); 13174 if (Val) 13175 return Val; 13176 } 13177 13178 // Turn STORE (BSWAP) -> sthbrx/stwbrx. 13179 if (cast<StoreSDNode>(N)->isUnindexed() && Opcode == ISD::BSWAP && 13180 N->getOperand(1).getNode()->hasOneUse() && 13181 (Op1VT == MVT::i32 || Op1VT == MVT::i16 || 13182 (Subtarget.hasLDBRX() && Subtarget.isPPC64() && Op1VT == MVT::i64))) { 13183 13184 // STBRX can only handle simple types and it makes no sense to store less 13185 // two bytes in byte-reversed order. 13186 EVT mVT = cast<StoreSDNode>(N)->getMemoryVT(); 13187 if (mVT.isExtended() || mVT.getSizeInBits() < 16) 13188 break; 13189 13190 SDValue BSwapOp = N->getOperand(1).getOperand(0); 13191 // Do an any-extend to 32-bits if this is a half-word input. 13192 if (BSwapOp.getValueType() == MVT::i16) 13193 BSwapOp = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, BSwapOp); 13194 13195 // If the type of BSWAP operand is wider than stored memory width 13196 // it need to be shifted to the right side before STBRX. 13197 if (Op1VT.bitsGT(mVT)) { 13198 int Shift = Op1VT.getSizeInBits() - mVT.getSizeInBits(); 13199 BSwapOp = DAG.getNode(ISD::SRL, dl, Op1VT, BSwapOp, 13200 DAG.getConstant(Shift, dl, MVT::i32)); 13201 // Need to truncate if this is a bswap of i64 stored as i32/i16. 13202 if (Op1VT == MVT::i64) 13203 BSwapOp = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, BSwapOp); 13204 } 13205 13206 SDValue Ops[] = { 13207 N->getOperand(0), BSwapOp, N->getOperand(2), DAG.getValueType(mVT) 13208 }; 13209 return 13210 DAG.getMemIntrinsicNode(PPCISD::STBRX, dl, DAG.getVTList(MVT::Other), 13211 Ops, cast<StoreSDNode>(N)->getMemoryVT(), 13212 cast<StoreSDNode>(N)->getMemOperand()); 13213 } 13214 13215 // STORE Constant:i32<0> -> STORE<trunc to i32> Constant:i64<0> 13216 // So it can increase the chance of CSE constant construction. 13217 if (Subtarget.isPPC64() && !DCI.isBeforeLegalize() && 13218 isa<ConstantSDNode>(N->getOperand(1)) && Op1VT == MVT::i32) { 13219 // Need to sign-extended to 64-bits to handle negative values. 13220 EVT MemVT = cast<StoreSDNode>(N)->getMemoryVT(); 13221 uint64_t Val64 = SignExtend64(N->getConstantOperandVal(1), 13222 MemVT.getSizeInBits()); 13223 SDValue Const64 = DAG.getConstant(Val64, dl, MVT::i64); 13224 13225 // DAG.getTruncStore() can't be used here because it doesn't accept 13226 // the general (base + offset) addressing mode. 13227 // So we use UpdateNodeOperands and setTruncatingStore instead. 13228 DAG.UpdateNodeOperands(N, N->getOperand(0), Const64, N->getOperand(2), 13229 N->getOperand(3)); 13230 cast<StoreSDNode>(N)->setTruncatingStore(true); 13231 return SDValue(N, 0); 13232 } 13233 13234 // For little endian, VSX stores require generating xxswapd/lxvd2x. 13235 // Not needed on ISA 3.0 based CPUs since we have a non-permuting store. 13236 if (Op1VT.isSimple()) { 13237 MVT StoreVT = Op1VT.getSimpleVT(); 13238 if (Subtarget.needsSwapsForVSXMemOps() && 13239 (StoreVT == MVT::v2f64 || StoreVT == MVT::v2i64 || 13240 StoreVT == MVT::v4f32 || StoreVT == MVT::v4i32)) 13241 return expandVSXStoreForLE(N, DCI); 13242 } 13243 break; 13244 } 13245 case ISD::LOAD: { 13246 LoadSDNode *LD = cast<LoadSDNode>(N); 13247 EVT VT = LD->getValueType(0); 13248 13249 // For little endian, VSX loads require generating lxvd2x/xxswapd. 13250 // Not needed on ISA 3.0 based CPUs since we have a non-permuting load. 13251 if (VT.isSimple()) { 13252 MVT LoadVT = VT.getSimpleVT(); 13253 if (Subtarget.needsSwapsForVSXMemOps() && 13254 (LoadVT == MVT::v2f64 || LoadVT == MVT::v2i64 || 13255 LoadVT == MVT::v4f32 || LoadVT == MVT::v4i32)) 13256 return expandVSXLoadForLE(N, DCI); 13257 } 13258 13259 // We sometimes end up with a 64-bit integer load, from which we extract 13260 // two single-precision floating-point numbers. This happens with 13261 // std::complex<float>, and other similar structures, because of the way we 13262 // canonicalize structure copies. However, if we lack direct moves, 13263 // then the final bitcasts from the extracted integer values to the 13264 // floating-point numbers turn into store/load pairs. Even with direct moves, 13265 // just loading the two floating-point numbers is likely better. 13266 auto ReplaceTwoFloatLoad = [&]() { 13267 if (VT != MVT::i64) 13268 return false; 13269 13270 if (LD->getExtensionType() != ISD::NON_EXTLOAD || 13271 LD->isVolatile()) 13272 return false; 13273 13274 // We're looking for a sequence like this: 13275 // t13: i64,ch = load<LD8[%ref.tmp]> t0, t6, undef:i64 13276 // t16: i64 = srl t13, Constant:i32<32> 13277 // t17: i32 = truncate t16 13278 // t18: f32 = bitcast t17 13279 // t19: i32 = truncate t13 13280 // t20: f32 = bitcast t19 13281 13282 if (!LD->hasNUsesOfValue(2, 0)) 13283 return false; 13284 13285 auto UI = LD->use_begin(); 13286 while (UI.getUse().getResNo() != 0) ++UI; 13287 SDNode *Trunc = *UI++; 13288 while (UI.getUse().getResNo() != 0) ++UI; 13289 SDNode *RightShift = *UI; 13290 if (Trunc->getOpcode() != ISD::TRUNCATE) 13291 std::swap(Trunc, RightShift); 13292 13293 if (Trunc->getOpcode() != ISD::TRUNCATE || 13294 Trunc->getValueType(0) != MVT::i32 || 13295 !Trunc->hasOneUse()) 13296 return false; 13297 if (RightShift->getOpcode() != ISD::SRL || 13298 !isa<ConstantSDNode>(RightShift->getOperand(1)) || 13299 RightShift->getConstantOperandVal(1) != 32 || 13300 !RightShift->hasOneUse()) 13301 return false; 13302 13303 SDNode *Trunc2 = *RightShift->use_begin(); 13304 if (Trunc2->getOpcode() != ISD::TRUNCATE || 13305 Trunc2->getValueType(0) != MVT::i32 || 13306 !Trunc2->hasOneUse()) 13307 return false; 13308 13309 SDNode *Bitcast = *Trunc->use_begin(); 13310 SDNode *Bitcast2 = *Trunc2->use_begin(); 13311 13312 if (Bitcast->getOpcode() != ISD::BITCAST || 13313 Bitcast->getValueType(0) != MVT::f32) 13314 return false; 13315 if (Bitcast2->getOpcode() != ISD::BITCAST || 13316 Bitcast2->getValueType(0) != MVT::f32) 13317 return false; 13318 13319 if (Subtarget.isLittleEndian()) 13320 std::swap(Bitcast, Bitcast2); 13321 13322 // Bitcast has the second float (in memory-layout order) and Bitcast2 13323 // has the first one. 13324 13325 SDValue BasePtr = LD->getBasePtr(); 13326 if (LD->isIndexed()) { 13327 assert(LD->getAddressingMode() == ISD::PRE_INC && 13328 "Non-pre-inc AM on PPC?"); 13329 BasePtr = 13330 DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 13331 LD->getOffset()); 13332 } 13333 13334 auto MMOFlags = 13335 LD->getMemOperand()->getFlags() & ~MachineMemOperand::MOVolatile; 13336 SDValue FloatLoad = DAG.getLoad(MVT::f32, dl, LD->getChain(), BasePtr, 13337 LD->getPointerInfo(), LD->getAlignment(), 13338 MMOFlags, LD->getAAInfo()); 13339 SDValue AddPtr = 13340 DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 13341 BasePtr, DAG.getIntPtrConstant(4, dl)); 13342 SDValue FloatLoad2 = DAG.getLoad( 13343 MVT::f32, dl, SDValue(FloatLoad.getNode(), 1), AddPtr, 13344 LD->getPointerInfo().getWithOffset(4), 13345 MinAlign(LD->getAlignment(), 4), MMOFlags, LD->getAAInfo()); 13346 13347 if (LD->isIndexed()) { 13348 // Note that DAGCombine should re-form any pre-increment load(s) from 13349 // what is produced here if that makes sense. 13350 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), BasePtr); 13351 } 13352 13353 DCI.CombineTo(Bitcast2, FloatLoad); 13354 DCI.CombineTo(Bitcast, FloatLoad2); 13355 13356 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, LD->isIndexed() ? 2 : 1), 13357 SDValue(FloatLoad2.getNode(), 1)); 13358 return true; 13359 }; 13360 13361 if (ReplaceTwoFloatLoad()) 13362 return SDValue(N, 0); 13363 13364 EVT MemVT = LD->getMemoryVT(); 13365 Type *Ty = MemVT.getTypeForEVT(*DAG.getContext()); 13366 unsigned ABIAlignment = DAG.getDataLayout().getABITypeAlignment(Ty); 13367 Type *STy = MemVT.getScalarType().getTypeForEVT(*DAG.getContext()); 13368 unsigned ScalarABIAlignment = DAG.getDataLayout().getABITypeAlignment(STy); 13369 if (LD->isUnindexed() && VT.isVector() && 13370 ((Subtarget.hasAltivec() && ISD::isNON_EXTLoad(N) && 13371 // P8 and later hardware should just use LOAD. 13372 !Subtarget.hasP8Vector() && (VT == MVT::v16i8 || VT == MVT::v8i16 || 13373 VT == MVT::v4i32 || VT == MVT::v4f32)) || 13374 (Subtarget.hasQPX() && (VT == MVT::v4f64 || VT == MVT::v4f32) && 13375 LD->getAlignment() >= ScalarABIAlignment)) && 13376 LD->getAlignment() < ABIAlignment) { 13377 // This is a type-legal unaligned Altivec or QPX load. 13378 SDValue Chain = LD->getChain(); 13379 SDValue Ptr = LD->getBasePtr(); 13380 bool isLittleEndian = Subtarget.isLittleEndian(); 13381 13382 // This implements the loading of unaligned vectors as described in 13383 // the venerable Apple Velocity Engine overview. Specifically: 13384 // https://developer.apple.com/hardwaredrivers/ve/alignment.html 13385 // https://developer.apple.com/hardwaredrivers/ve/code_optimization.html 13386 // 13387 // The general idea is to expand a sequence of one or more unaligned 13388 // loads into an alignment-based permutation-control instruction (lvsl 13389 // or lvsr), a series of regular vector loads (which always truncate 13390 // their input address to an aligned address), and a series of 13391 // permutations. The results of these permutations are the requested 13392 // loaded values. The trick is that the last "extra" load is not taken 13393 // from the address you might suspect (sizeof(vector) bytes after the 13394 // last requested load), but rather sizeof(vector) - 1 bytes after the 13395 // last requested vector. The point of this is to avoid a page fault if 13396 // the base address happened to be aligned. This works because if the 13397 // base address is aligned, then adding less than a full vector length 13398 // will cause the last vector in the sequence to be (re)loaded. 13399 // Otherwise, the next vector will be fetched as you might suspect was 13400 // necessary. 13401 13402 // We might be able to reuse the permutation generation from 13403 // a different base address offset from this one by an aligned amount. 13404 // The INTRINSIC_WO_CHAIN DAG combine will attempt to perform this 13405 // optimization later. 13406 Intrinsic::ID Intr, IntrLD, IntrPerm; 13407 MVT PermCntlTy, PermTy, LDTy; 13408 if (Subtarget.hasAltivec()) { 13409 Intr = isLittleEndian ? Intrinsic::ppc_altivec_lvsr : 13410 Intrinsic::ppc_altivec_lvsl; 13411 IntrLD = Intrinsic::ppc_altivec_lvx; 13412 IntrPerm = Intrinsic::ppc_altivec_vperm; 13413 PermCntlTy = MVT::v16i8; 13414 PermTy = MVT::v4i32; 13415 LDTy = MVT::v4i32; 13416 } else { 13417 Intr = MemVT == MVT::v4f64 ? Intrinsic::ppc_qpx_qvlpcld : 13418 Intrinsic::ppc_qpx_qvlpcls; 13419 IntrLD = MemVT == MVT::v4f64 ? Intrinsic::ppc_qpx_qvlfd : 13420 Intrinsic::ppc_qpx_qvlfs; 13421 IntrPerm = Intrinsic::ppc_qpx_qvfperm; 13422 PermCntlTy = MVT::v4f64; 13423 PermTy = MVT::v4f64; 13424 LDTy = MemVT.getSimpleVT(); 13425 } 13426 13427 SDValue PermCntl = BuildIntrinsicOp(Intr, Ptr, DAG, dl, PermCntlTy); 13428 13429 // Create the new MMO for the new base load. It is like the original MMO, 13430 // but represents an area in memory almost twice the vector size centered 13431 // on the original address. If the address is unaligned, we might start 13432 // reading up to (sizeof(vector)-1) bytes below the address of the 13433 // original unaligned load. 13434 MachineFunction &MF = DAG.getMachineFunction(); 13435 MachineMemOperand *BaseMMO = 13436 MF.getMachineMemOperand(LD->getMemOperand(), 13437 -(long)MemVT.getStoreSize()+1, 13438 2*MemVT.getStoreSize()-1); 13439 13440 // Create the new base load. 13441 SDValue LDXIntID = 13442 DAG.getTargetConstant(IntrLD, dl, getPointerTy(MF.getDataLayout())); 13443 SDValue BaseLoadOps[] = { Chain, LDXIntID, Ptr }; 13444 SDValue BaseLoad = 13445 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, dl, 13446 DAG.getVTList(PermTy, MVT::Other), 13447 BaseLoadOps, LDTy, BaseMMO); 13448 13449 // Note that the value of IncOffset (which is provided to the next 13450 // load's pointer info offset value, and thus used to calculate the 13451 // alignment), and the value of IncValue (which is actually used to 13452 // increment the pointer value) are different! This is because we 13453 // require the next load to appear to be aligned, even though it 13454 // is actually offset from the base pointer by a lesser amount. 13455 int IncOffset = VT.getSizeInBits() / 8; 13456 int IncValue = IncOffset; 13457 13458 // Walk (both up and down) the chain looking for another load at the real 13459 // (aligned) offset (the alignment of the other load does not matter in 13460 // this case). If found, then do not use the offset reduction trick, as 13461 // that will prevent the loads from being later combined (as they would 13462 // otherwise be duplicates). 13463 if (!findConsecutiveLoad(LD, DAG)) 13464 --IncValue; 13465 13466 SDValue Increment = 13467 DAG.getConstant(IncValue, dl, getPointerTy(MF.getDataLayout())); 13468 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); 13469 13470 MachineMemOperand *ExtraMMO = 13471 MF.getMachineMemOperand(LD->getMemOperand(), 13472 1, 2*MemVT.getStoreSize()-1); 13473 SDValue ExtraLoadOps[] = { Chain, LDXIntID, Ptr }; 13474 SDValue ExtraLoad = 13475 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, dl, 13476 DAG.getVTList(PermTy, MVT::Other), 13477 ExtraLoadOps, LDTy, ExtraMMO); 13478 13479 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 13480 BaseLoad.getValue(1), ExtraLoad.getValue(1)); 13481 13482 // Because vperm has a big-endian bias, we must reverse the order 13483 // of the input vectors and complement the permute control vector 13484 // when generating little endian code. We have already handled the 13485 // latter by using lvsr instead of lvsl, so just reverse BaseLoad 13486 // and ExtraLoad here. 13487 SDValue Perm; 13488 if (isLittleEndian) 13489 Perm = BuildIntrinsicOp(IntrPerm, 13490 ExtraLoad, BaseLoad, PermCntl, DAG, dl); 13491 else 13492 Perm = BuildIntrinsicOp(IntrPerm, 13493 BaseLoad, ExtraLoad, PermCntl, DAG, dl); 13494 13495 if (VT != PermTy) 13496 Perm = Subtarget.hasAltivec() ? 13497 DAG.getNode(ISD::BITCAST, dl, VT, Perm) : 13498 DAG.getNode(ISD::FP_ROUND, dl, VT, Perm, // QPX 13499 DAG.getTargetConstant(1, dl, MVT::i64)); 13500 // second argument is 1 because this rounding 13501 // is always exact. 13502 13503 // The output of the permutation is our loaded result, the TokenFactor is 13504 // our new chain. 13505 DCI.CombineTo(N, Perm, TF); 13506 return SDValue(N, 0); 13507 } 13508 } 13509 break; 13510 case ISD::INTRINSIC_WO_CHAIN: { 13511 bool isLittleEndian = Subtarget.isLittleEndian(); 13512 unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 13513 Intrinsic::ID Intr = (isLittleEndian ? Intrinsic::ppc_altivec_lvsr 13514 : Intrinsic::ppc_altivec_lvsl); 13515 if ((IID == Intr || 13516 IID == Intrinsic::ppc_qpx_qvlpcld || 13517 IID == Intrinsic::ppc_qpx_qvlpcls) && 13518 N->getOperand(1)->getOpcode() == ISD::ADD) { 13519 SDValue Add = N->getOperand(1); 13520 13521 int Bits = IID == Intrinsic::ppc_qpx_qvlpcld ? 13522 5 /* 32 byte alignment */ : 4 /* 16 byte alignment */; 13523 13524 if (DAG.MaskedValueIsZero(Add->getOperand(1), 13525 APInt::getAllOnesValue(Bits /* alignment */) 13526 .zext(Add.getScalarValueSizeInBits()))) { 13527 SDNode *BasePtr = Add->getOperand(0).getNode(); 13528 for (SDNode::use_iterator UI = BasePtr->use_begin(), 13529 UE = BasePtr->use_end(); 13530 UI != UE; ++UI) { 13531 if (UI->getOpcode() == ISD::INTRINSIC_WO_CHAIN && 13532 cast<ConstantSDNode>(UI->getOperand(0))->getZExtValue() == IID) { 13533 // We've found another LVSL/LVSR, and this address is an aligned 13534 // multiple of that one. The results will be the same, so use the 13535 // one we've just found instead. 13536 13537 return SDValue(*UI, 0); 13538 } 13539 } 13540 } 13541 13542 if (isa<ConstantSDNode>(Add->getOperand(1))) { 13543 SDNode *BasePtr = Add->getOperand(0).getNode(); 13544 for (SDNode::use_iterator UI = BasePtr->use_begin(), 13545 UE = BasePtr->use_end(); UI != UE; ++UI) { 13546 if (UI->getOpcode() == ISD::ADD && 13547 isa<ConstantSDNode>(UI->getOperand(1)) && 13548 (cast<ConstantSDNode>(Add->getOperand(1))->getZExtValue() - 13549 cast<ConstantSDNode>(UI->getOperand(1))->getZExtValue()) % 13550 (1ULL << Bits) == 0) { 13551 SDNode *OtherAdd = *UI; 13552 for (SDNode::use_iterator VI = OtherAdd->use_begin(), 13553 VE = OtherAdd->use_end(); VI != VE; ++VI) { 13554 if (VI->getOpcode() == ISD::INTRINSIC_WO_CHAIN && 13555 cast<ConstantSDNode>(VI->getOperand(0))->getZExtValue() == IID) { 13556 return SDValue(*VI, 0); 13557 } 13558 } 13559 } 13560 } 13561 } 13562 } 13563 13564 // Combine vmaxsw/h/b(a, a's negation) to abs(a) 13565 // Expose the vabsduw/h/b opportunity for down stream 13566 if (!DCI.isAfterLegalizeDAG() && Subtarget.hasP9Altivec() && 13567 (IID == Intrinsic::ppc_altivec_vmaxsw || 13568 IID == Intrinsic::ppc_altivec_vmaxsh || 13569 IID == Intrinsic::ppc_altivec_vmaxsb)) { 13570 SDValue V1 = N->getOperand(1); 13571 SDValue V2 = N->getOperand(2); 13572 if ((V1.getSimpleValueType() == MVT::v4i32 || 13573 V1.getSimpleValueType() == MVT::v8i16 || 13574 V1.getSimpleValueType() == MVT::v16i8) && 13575 V1.getSimpleValueType() == V2.getSimpleValueType()) { 13576 // (0-a, a) 13577 if (V1.getOpcode() == ISD::SUB && 13578 ISD::isBuildVectorAllZeros(V1.getOperand(0).getNode()) && 13579 V1.getOperand(1) == V2) { 13580 return DAG.getNode(ISD::ABS, dl, V2.getValueType(), V2); 13581 } 13582 // (a, 0-a) 13583 if (V2.getOpcode() == ISD::SUB && 13584 ISD::isBuildVectorAllZeros(V2.getOperand(0).getNode()) && 13585 V2.getOperand(1) == V1) { 13586 return DAG.getNode(ISD::ABS, dl, V1.getValueType(), V1); 13587 } 13588 // (x-y, y-x) 13589 if (V1.getOpcode() == ISD::SUB && V2.getOpcode() == ISD::SUB && 13590 V1.getOperand(0) == V2.getOperand(1) && 13591 V1.getOperand(1) == V2.getOperand(0)) { 13592 return DAG.getNode(ISD::ABS, dl, V1.getValueType(), V1); 13593 } 13594 } 13595 } 13596 } 13597 13598 break; 13599 case ISD::INTRINSIC_W_CHAIN: 13600 // For little endian, VSX loads require generating lxvd2x/xxswapd. 13601 // Not needed on ISA 3.0 based CPUs since we have a non-permuting load. 13602 if (Subtarget.needsSwapsForVSXMemOps()) { 13603 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) { 13604 default: 13605 break; 13606 case Intrinsic::ppc_vsx_lxvw4x: 13607 case Intrinsic::ppc_vsx_lxvd2x: 13608 return expandVSXLoadForLE(N, DCI); 13609 } 13610 } 13611 break; 13612 case ISD::INTRINSIC_VOID: 13613 // For little endian, VSX stores require generating xxswapd/stxvd2x. 13614 // Not needed on ISA 3.0 based CPUs since we have a non-permuting store. 13615 if (Subtarget.needsSwapsForVSXMemOps()) { 13616 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) { 13617 default: 13618 break; 13619 case Intrinsic::ppc_vsx_stxvw4x: 13620 case Intrinsic::ppc_vsx_stxvd2x: 13621 return expandVSXStoreForLE(N, DCI); 13622 } 13623 } 13624 break; 13625 case ISD::BSWAP: 13626 // Turn BSWAP (LOAD) -> lhbrx/lwbrx. 13627 if (ISD::isNON_EXTLoad(N->getOperand(0).getNode()) && 13628 N->getOperand(0).hasOneUse() && 13629 (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i16 || 13630 (Subtarget.hasLDBRX() && Subtarget.isPPC64() && 13631 N->getValueType(0) == MVT::i64))) { 13632 SDValue Load = N->getOperand(0); 13633 LoadSDNode *LD = cast<LoadSDNode>(Load); 13634 // Create the byte-swapping load. 13635 SDValue Ops[] = { 13636 LD->getChain(), // Chain 13637 LD->getBasePtr(), // Ptr 13638 DAG.getValueType(N->getValueType(0)) // VT 13639 }; 13640 SDValue BSLoad = 13641 DAG.getMemIntrinsicNode(PPCISD::LBRX, dl, 13642 DAG.getVTList(N->getValueType(0) == MVT::i64 ? 13643 MVT::i64 : MVT::i32, MVT::Other), 13644 Ops, LD->getMemoryVT(), LD->getMemOperand()); 13645 13646 // If this is an i16 load, insert the truncate. 13647 SDValue ResVal = BSLoad; 13648 if (N->getValueType(0) == MVT::i16) 13649 ResVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, BSLoad); 13650 13651 // First, combine the bswap away. This makes the value produced by the 13652 // load dead. 13653 DCI.CombineTo(N, ResVal); 13654 13655 // Next, combine the load away, we give it a bogus result value but a real 13656 // chain result. The result value is dead because the bswap is dead. 13657 DCI.CombineTo(Load.getNode(), ResVal, BSLoad.getValue(1)); 13658 13659 // Return N so it doesn't get rechecked! 13660 return SDValue(N, 0); 13661 } 13662 break; 13663 case PPCISD::VCMP: 13664 // If a VCMPo node already exists with exactly the same operands as this 13665 // node, use its result instead of this node (VCMPo computes both a CR6 and 13666 // a normal output). 13667 // 13668 if (!N->getOperand(0).hasOneUse() && 13669 !N->getOperand(1).hasOneUse() && 13670 !N->getOperand(2).hasOneUse()) { 13671 13672 // Scan all of the users of the LHS, looking for VCMPo's that match. 13673 SDNode *VCMPoNode = nullptr; 13674 13675 SDNode *LHSN = N->getOperand(0).getNode(); 13676 for (SDNode::use_iterator UI = LHSN->use_begin(), E = LHSN->use_end(); 13677 UI != E; ++UI) 13678 if (UI->getOpcode() == PPCISD::VCMPo && 13679 UI->getOperand(1) == N->getOperand(1) && 13680 UI->getOperand(2) == N->getOperand(2) && 13681 UI->getOperand(0) == N->getOperand(0)) { 13682 VCMPoNode = *UI; 13683 break; 13684 } 13685 13686 // If there is no VCMPo node, or if the flag value has a single use, don't 13687 // transform this. 13688 if (!VCMPoNode || VCMPoNode->hasNUsesOfValue(0, 1)) 13689 break; 13690 13691 // Look at the (necessarily single) use of the flag value. If it has a 13692 // chain, this transformation is more complex. Note that multiple things 13693 // could use the value result, which we should ignore. 13694 SDNode *FlagUser = nullptr; 13695 for (SDNode::use_iterator UI = VCMPoNode->use_begin(); 13696 FlagUser == nullptr; ++UI) { 13697 assert(UI != VCMPoNode->use_end() && "Didn't find user!"); 13698 SDNode *User = *UI; 13699 for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) { 13700 if (User->getOperand(i) == SDValue(VCMPoNode, 1)) { 13701 FlagUser = User; 13702 break; 13703 } 13704 } 13705 } 13706 13707 // If the user is a MFOCRF instruction, we know this is safe. 13708 // Otherwise we give up for right now. 13709 if (FlagUser->getOpcode() == PPCISD::MFOCRF) 13710 return SDValue(VCMPoNode, 0); 13711 } 13712 break; 13713 case ISD::BRCOND: { 13714 SDValue Cond = N->getOperand(1); 13715 SDValue Target = N->getOperand(2); 13716 13717 if (Cond.getOpcode() == ISD::INTRINSIC_W_CHAIN && 13718 cast<ConstantSDNode>(Cond.getOperand(1))->getZExtValue() == 13719 Intrinsic::loop_decrement) { 13720 13721 // We now need to make the intrinsic dead (it cannot be instruction 13722 // selected). 13723 DAG.ReplaceAllUsesOfValueWith(Cond.getValue(1), Cond.getOperand(0)); 13724 assert(Cond.getNode()->hasOneUse() && 13725 "Counter decrement has more than one use"); 13726 13727 return DAG.getNode(PPCISD::BDNZ, dl, MVT::Other, 13728 N->getOperand(0), Target); 13729 } 13730 } 13731 break; 13732 case ISD::BR_CC: { 13733 // If this is a branch on an altivec predicate comparison, lower this so 13734 // that we don't have to do a MFOCRF: instead, branch directly on CR6. This 13735 // lowering is done pre-legalize, because the legalizer lowers the predicate 13736 // compare down to code that is difficult to reassemble. 13737 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get(); 13738 SDValue LHS = N->getOperand(2), RHS = N->getOperand(3); 13739 13740 // Sometimes the promoted value of the intrinsic is ANDed by some non-zero 13741 // value. If so, pass-through the AND to get to the intrinsic. 13742 if (LHS.getOpcode() == ISD::AND && 13743 LHS.getOperand(0).getOpcode() == ISD::INTRINSIC_W_CHAIN && 13744 cast<ConstantSDNode>(LHS.getOperand(0).getOperand(1))->getZExtValue() == 13745 Intrinsic::loop_decrement && 13746 isa<ConstantSDNode>(LHS.getOperand(1)) && 13747 !isNullConstant(LHS.getOperand(1))) 13748 LHS = LHS.getOperand(0); 13749 13750 if (LHS.getOpcode() == ISD::INTRINSIC_W_CHAIN && 13751 cast<ConstantSDNode>(LHS.getOperand(1))->getZExtValue() == 13752 Intrinsic::loop_decrement && 13753 isa<ConstantSDNode>(RHS)) { 13754 assert((CC == ISD::SETEQ || CC == ISD::SETNE) && 13755 "Counter decrement comparison is not EQ or NE"); 13756 13757 unsigned Val = cast<ConstantSDNode>(RHS)->getZExtValue(); 13758 bool isBDNZ = (CC == ISD::SETEQ && Val) || 13759 (CC == ISD::SETNE && !Val); 13760 13761 // We now need to make the intrinsic dead (it cannot be instruction 13762 // selected). 13763 DAG.ReplaceAllUsesOfValueWith(LHS.getValue(1), LHS.getOperand(0)); 13764 assert(LHS.getNode()->hasOneUse() && 13765 "Counter decrement has more than one use"); 13766 13767 return DAG.getNode(isBDNZ ? PPCISD::BDNZ : PPCISD::BDZ, dl, MVT::Other, 13768 N->getOperand(0), N->getOperand(4)); 13769 } 13770 13771 int CompareOpc; 13772 bool isDot; 13773 13774 if (LHS.getOpcode() == ISD::INTRINSIC_WO_CHAIN && 13775 isa<ConstantSDNode>(RHS) && (CC == ISD::SETEQ || CC == ISD::SETNE) && 13776 getVectorCompareInfo(LHS, CompareOpc, isDot, Subtarget)) { 13777 assert(isDot && "Can't compare against a vector result!"); 13778 13779 // If this is a comparison against something other than 0/1, then we know 13780 // that the condition is never/always true. 13781 unsigned Val = cast<ConstantSDNode>(RHS)->getZExtValue(); 13782 if (Val != 0 && Val != 1) { 13783 if (CC == ISD::SETEQ) // Cond never true, remove branch. 13784 return N->getOperand(0); 13785 // Always !=, turn it into an unconditional branch. 13786 return DAG.getNode(ISD::BR, dl, MVT::Other, 13787 N->getOperand(0), N->getOperand(4)); 13788 } 13789 13790 bool BranchOnWhenPredTrue = (CC == ISD::SETEQ) ^ (Val == 0); 13791 13792 // Create the PPCISD altivec 'dot' comparison node. 13793 SDValue Ops[] = { 13794 LHS.getOperand(2), // LHS of compare 13795 LHS.getOperand(3), // RHS of compare 13796 DAG.getConstant(CompareOpc, dl, MVT::i32) 13797 }; 13798 EVT VTs[] = { LHS.getOperand(2).getValueType(), MVT::Glue }; 13799 SDValue CompNode = DAG.getNode(PPCISD::VCMPo, dl, VTs, Ops); 13800 13801 // Unpack the result based on how the target uses it. 13802 PPC::Predicate CompOpc; 13803 switch (cast<ConstantSDNode>(LHS.getOperand(1))->getZExtValue()) { 13804 default: // Can't happen, don't crash on invalid number though. 13805 case 0: // Branch on the value of the EQ bit of CR6. 13806 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_EQ : PPC::PRED_NE; 13807 break; 13808 case 1: // Branch on the inverted value of the EQ bit of CR6. 13809 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_NE : PPC::PRED_EQ; 13810 break; 13811 case 2: // Branch on the value of the LT bit of CR6. 13812 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_LT : PPC::PRED_GE; 13813 break; 13814 case 3: // Branch on the inverted value of the LT bit of CR6. 13815 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_GE : PPC::PRED_LT; 13816 break; 13817 } 13818 13819 return DAG.getNode(PPCISD::COND_BRANCH, dl, MVT::Other, N->getOperand(0), 13820 DAG.getConstant(CompOpc, dl, MVT::i32), 13821 DAG.getRegister(PPC::CR6, MVT::i32), 13822 N->getOperand(4), CompNode.getValue(1)); 13823 } 13824 break; 13825 } 13826 case ISD::BUILD_VECTOR: 13827 return DAGCombineBuildVector(N, DCI); 13828 case ISD::ABS: 13829 return combineABS(N, DCI); 13830 case ISD::VSELECT: 13831 return combineVSelect(N, DCI); 13832 } 13833 13834 return SDValue(); 13835 } 13836 13837 SDValue 13838 PPCTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor, 13839 SelectionDAG &DAG, 13840 SmallVectorImpl<SDNode *> &Created) const { 13841 // fold (sdiv X, pow2) 13842 EVT VT = N->getValueType(0); 13843 if (VT == MVT::i64 && !Subtarget.isPPC64()) 13844 return SDValue(); 13845 if ((VT != MVT::i32 && VT != MVT::i64) || 13846 !(Divisor.isPowerOf2() || (-Divisor).isPowerOf2())) 13847 return SDValue(); 13848 13849 SDLoc DL(N); 13850 SDValue N0 = N->getOperand(0); 13851 13852 bool IsNegPow2 = (-Divisor).isPowerOf2(); 13853 unsigned Lg2 = (IsNegPow2 ? -Divisor : Divisor).countTrailingZeros(); 13854 SDValue ShiftAmt = DAG.getConstant(Lg2, DL, VT); 13855 13856 SDValue Op = DAG.getNode(PPCISD::SRA_ADDZE, DL, VT, N0, ShiftAmt); 13857 Created.push_back(Op.getNode()); 13858 13859 if (IsNegPow2) { 13860 Op = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op); 13861 Created.push_back(Op.getNode()); 13862 } 13863 13864 return Op; 13865 } 13866 13867 //===----------------------------------------------------------------------===// 13868 // Inline Assembly Support 13869 //===----------------------------------------------------------------------===// 13870 13871 void PPCTargetLowering::computeKnownBitsForTargetNode(const SDValue Op, 13872 KnownBits &Known, 13873 const APInt &DemandedElts, 13874 const SelectionDAG &DAG, 13875 unsigned Depth) const { 13876 Known.resetAll(); 13877 switch (Op.getOpcode()) { 13878 default: break; 13879 case PPCISD::LBRX: { 13880 // lhbrx is known to have the top bits cleared out. 13881 if (cast<VTSDNode>(Op.getOperand(2))->getVT() == MVT::i16) 13882 Known.Zero = 0xFFFF0000; 13883 break; 13884 } 13885 case ISD::INTRINSIC_WO_CHAIN: { 13886 switch (cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue()) { 13887 default: break; 13888 case Intrinsic::ppc_altivec_vcmpbfp_p: 13889 case Intrinsic::ppc_altivec_vcmpeqfp_p: 13890 case Intrinsic::ppc_altivec_vcmpequb_p: 13891 case Intrinsic::ppc_altivec_vcmpequh_p: 13892 case Intrinsic::ppc_altivec_vcmpequw_p: 13893 case Intrinsic::ppc_altivec_vcmpequd_p: 13894 case Intrinsic::ppc_altivec_vcmpgefp_p: 13895 case Intrinsic::ppc_altivec_vcmpgtfp_p: 13896 case Intrinsic::ppc_altivec_vcmpgtsb_p: 13897 case Intrinsic::ppc_altivec_vcmpgtsh_p: 13898 case Intrinsic::ppc_altivec_vcmpgtsw_p: 13899 case Intrinsic::ppc_altivec_vcmpgtsd_p: 13900 case Intrinsic::ppc_altivec_vcmpgtub_p: 13901 case Intrinsic::ppc_altivec_vcmpgtuh_p: 13902 case Intrinsic::ppc_altivec_vcmpgtuw_p: 13903 case Intrinsic::ppc_altivec_vcmpgtud_p: 13904 Known.Zero = ~1U; // All bits but the low one are known to be zero. 13905 break; 13906 } 13907 } 13908 } 13909 } 13910 13911 unsigned PPCTargetLowering::getPrefLoopAlignment(MachineLoop *ML) const { 13912 switch (Subtarget.getDarwinDirective()) { 13913 default: break; 13914 case PPC::DIR_970: 13915 case PPC::DIR_PWR4: 13916 case PPC::DIR_PWR5: 13917 case PPC::DIR_PWR5X: 13918 case PPC::DIR_PWR6: 13919 case PPC::DIR_PWR6X: 13920 case PPC::DIR_PWR7: 13921 case PPC::DIR_PWR8: 13922 case PPC::DIR_PWR9: { 13923 if (!ML) 13924 break; 13925 13926 if (!DisableInnermostLoopAlign32) { 13927 // If the nested loop is an innermost loop, prefer to a 32-byte alignment, 13928 // so that we can decrease cache misses and branch-prediction misses. 13929 // Actual alignment of the loop will depend on the hotness check and other 13930 // logic in alignBlocks. 13931 if (ML->getLoopDepth() > 1 && ML->getSubLoops().empty()) 13932 return 5; 13933 } 13934 13935 const PPCInstrInfo *TII = Subtarget.getInstrInfo(); 13936 13937 // For small loops (between 5 and 8 instructions), align to a 32-byte 13938 // boundary so that the entire loop fits in one instruction-cache line. 13939 uint64_t LoopSize = 0; 13940 for (auto I = ML->block_begin(), IE = ML->block_end(); I != IE; ++I) 13941 for (auto J = (*I)->begin(), JE = (*I)->end(); J != JE; ++J) { 13942 LoopSize += TII->getInstSizeInBytes(*J); 13943 if (LoopSize > 32) 13944 break; 13945 } 13946 13947 if (LoopSize > 16 && LoopSize <= 32) 13948 return 5; 13949 13950 break; 13951 } 13952 } 13953 13954 return TargetLowering::getPrefLoopAlignment(ML); 13955 } 13956 13957 /// getConstraintType - Given a constraint, return the type of 13958 /// constraint it is for this target. 13959 PPCTargetLowering::ConstraintType 13960 PPCTargetLowering::getConstraintType(StringRef Constraint) const { 13961 if (Constraint.size() == 1) { 13962 switch (Constraint[0]) { 13963 default: break; 13964 case 'b': 13965 case 'r': 13966 case 'f': 13967 case 'd': 13968 case 'v': 13969 case 'y': 13970 return C_RegisterClass; 13971 case 'Z': 13972 // FIXME: While Z does indicate a memory constraint, it specifically 13973 // indicates an r+r address (used in conjunction with the 'y' modifier 13974 // in the replacement string). Currently, we're forcing the base 13975 // register to be r0 in the asm printer (which is interpreted as zero) 13976 // and forming the complete address in the second register. This is 13977 // suboptimal. 13978 return C_Memory; 13979 } 13980 } else if (Constraint == "wc") { // individual CR bits. 13981 return C_RegisterClass; 13982 } else if (Constraint == "wa" || Constraint == "wd" || 13983 Constraint == "wf" || Constraint == "ws" || 13984 Constraint == "wi" || Constraint == "ww") { 13985 return C_RegisterClass; // VSX registers. 13986 } 13987 return TargetLowering::getConstraintType(Constraint); 13988 } 13989 13990 /// Examine constraint type and operand type and determine a weight value. 13991 /// This object must already have been set up with the operand type 13992 /// and the current alternative constraint selected. 13993 TargetLowering::ConstraintWeight 13994 PPCTargetLowering::getSingleConstraintMatchWeight( 13995 AsmOperandInfo &info, const char *constraint) const { 13996 ConstraintWeight weight = CW_Invalid; 13997 Value *CallOperandVal = info.CallOperandVal; 13998 // If we don't have a value, we can't do a match, 13999 // but allow it at the lowest weight. 14000 if (!CallOperandVal) 14001 return CW_Default; 14002 Type *type = CallOperandVal->getType(); 14003 14004 // Look at the constraint type. 14005 if (StringRef(constraint) == "wc" && type->isIntegerTy(1)) 14006 return CW_Register; // an individual CR bit. 14007 else if ((StringRef(constraint) == "wa" || 14008 StringRef(constraint) == "wd" || 14009 StringRef(constraint) == "wf") && 14010 type->isVectorTy()) 14011 return CW_Register; 14012 else if (StringRef(constraint) == "wi" && type->isIntegerTy(64)) 14013 return CW_Register; // just hold 64-bit integers data. 14014 else if (StringRef(constraint) == "ws" && type->isDoubleTy()) 14015 return CW_Register; 14016 else if (StringRef(constraint) == "ww" && type->isFloatTy()) 14017 return CW_Register; 14018 14019 switch (*constraint) { 14020 default: 14021 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); 14022 break; 14023 case 'b': 14024 if (type->isIntegerTy()) 14025 weight = CW_Register; 14026 break; 14027 case 'f': 14028 if (type->isFloatTy()) 14029 weight = CW_Register; 14030 break; 14031 case 'd': 14032 if (type->isDoubleTy()) 14033 weight = CW_Register; 14034 break; 14035 case 'v': 14036 if (type->isVectorTy()) 14037 weight = CW_Register; 14038 break; 14039 case 'y': 14040 weight = CW_Register; 14041 break; 14042 case 'Z': 14043 weight = CW_Memory; 14044 break; 14045 } 14046 return weight; 14047 } 14048 14049 std::pair<unsigned, const TargetRegisterClass *> 14050 PPCTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, 14051 StringRef Constraint, 14052 MVT VT) const { 14053 if (Constraint.size() == 1) { 14054 // GCC RS6000 Constraint Letters 14055 switch (Constraint[0]) { 14056 case 'b': // R1-R31 14057 if (VT == MVT::i64 && Subtarget.isPPC64()) 14058 return std::make_pair(0U, &PPC::G8RC_NOX0RegClass); 14059 return std::make_pair(0U, &PPC::GPRC_NOR0RegClass); 14060 case 'r': // R0-R31 14061 if (VT == MVT::i64 && Subtarget.isPPC64()) 14062 return std::make_pair(0U, &PPC::G8RCRegClass); 14063 return std::make_pair(0U, &PPC::GPRCRegClass); 14064 // 'd' and 'f' constraints are both defined to be "the floating point 14065 // registers", where one is for 32-bit and the other for 64-bit. We don't 14066 // really care overly much here so just give them all the same reg classes. 14067 case 'd': 14068 case 'f': 14069 if (Subtarget.hasSPE()) { 14070 if (VT == MVT::f32 || VT == MVT::i32) 14071 return std::make_pair(0U, &PPC::SPE4RCRegClass); 14072 if (VT == MVT::f64 || VT == MVT::i64) 14073 return std::make_pair(0U, &PPC::SPERCRegClass); 14074 } else { 14075 if (VT == MVT::f32 || VT == MVT::i32) 14076 return std::make_pair(0U, &PPC::F4RCRegClass); 14077 if (VT == MVT::f64 || VT == MVT::i64) 14078 return std::make_pair(0U, &PPC::F8RCRegClass); 14079 if (VT == MVT::v4f64 && Subtarget.hasQPX()) 14080 return std::make_pair(0U, &PPC::QFRCRegClass); 14081 if (VT == MVT::v4f32 && Subtarget.hasQPX()) 14082 return std::make_pair(0U, &PPC::QSRCRegClass); 14083 } 14084 break; 14085 case 'v': 14086 if (VT == MVT::v4f64 && Subtarget.hasQPX()) 14087 return std::make_pair(0U, &PPC::QFRCRegClass); 14088 if (VT == MVT::v4f32 && Subtarget.hasQPX()) 14089 return std::make_pair(0U, &PPC::QSRCRegClass); 14090 if (Subtarget.hasAltivec()) 14091 return std::make_pair(0U, &PPC::VRRCRegClass); 14092 break; 14093 case 'y': // crrc 14094 return std::make_pair(0U, &PPC::CRRCRegClass); 14095 } 14096 } else if (Constraint == "wc" && Subtarget.useCRBits()) { 14097 // An individual CR bit. 14098 return std::make_pair(0U, &PPC::CRBITRCRegClass); 14099 } else if ((Constraint == "wa" || Constraint == "wd" || 14100 Constraint == "wf" || Constraint == "wi") && 14101 Subtarget.hasVSX()) { 14102 return std::make_pair(0U, &PPC::VSRCRegClass); 14103 } else if ((Constraint == "ws" || Constraint == "ww") && Subtarget.hasVSX()) { 14104 if (VT == MVT::f32 && Subtarget.hasP8Vector()) 14105 return std::make_pair(0U, &PPC::VSSRCRegClass); 14106 else 14107 return std::make_pair(0U, &PPC::VSFRCRegClass); 14108 } 14109 14110 std::pair<unsigned, const TargetRegisterClass *> R = 14111 TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); 14112 14113 // r[0-9]+ are used, on PPC64, to refer to the corresponding 64-bit registers 14114 // (which we call X[0-9]+). If a 64-bit value has been requested, and a 14115 // 32-bit GPR has been selected, then 'upgrade' it to the 64-bit parent 14116 // register. 14117 // FIXME: If TargetLowering::getRegForInlineAsmConstraint could somehow use 14118 // the AsmName field from *RegisterInfo.td, then this would not be necessary. 14119 if (R.first && VT == MVT::i64 && Subtarget.isPPC64() && 14120 PPC::GPRCRegClass.contains(R.first)) 14121 return std::make_pair(TRI->getMatchingSuperReg(R.first, 14122 PPC::sub_32, &PPC::G8RCRegClass), 14123 &PPC::G8RCRegClass); 14124 14125 // GCC accepts 'cc' as an alias for 'cr0', and we need to do the same. 14126 if (!R.second && StringRef("{cc}").equals_lower(Constraint)) { 14127 R.first = PPC::CR0; 14128 R.second = &PPC::CRRCRegClass; 14129 } 14130 14131 return R; 14132 } 14133 14134 /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops 14135 /// vector. If it is invalid, don't add anything to Ops. 14136 void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op, 14137 std::string &Constraint, 14138 std::vector<SDValue>&Ops, 14139 SelectionDAG &DAG) const { 14140 SDValue Result; 14141 14142 // Only support length 1 constraints. 14143 if (Constraint.length() > 1) return; 14144 14145 char Letter = Constraint[0]; 14146 switch (Letter) { 14147 default: break; 14148 case 'I': 14149 case 'J': 14150 case 'K': 14151 case 'L': 14152 case 'M': 14153 case 'N': 14154 case 'O': 14155 case 'P': { 14156 ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op); 14157 if (!CST) return; // Must be an immediate to match. 14158 SDLoc dl(Op); 14159 int64_t Value = CST->getSExtValue(); 14160 EVT TCVT = MVT::i64; // All constants taken to be 64 bits so that negative 14161 // numbers are printed as such. 14162 switch (Letter) { 14163 default: llvm_unreachable("Unknown constraint letter!"); 14164 case 'I': // "I" is a signed 16-bit constant. 14165 if (isInt<16>(Value)) 14166 Result = DAG.getTargetConstant(Value, dl, TCVT); 14167 break; 14168 case 'J': // "J" is a constant with only the high-order 16 bits nonzero. 14169 if (isShiftedUInt<16, 16>(Value)) 14170 Result = DAG.getTargetConstant(Value, dl, TCVT); 14171 break; 14172 case 'L': // "L" is a signed 16-bit constant shifted left 16 bits. 14173 if (isShiftedInt<16, 16>(Value)) 14174 Result = DAG.getTargetConstant(Value, dl, TCVT); 14175 break; 14176 case 'K': // "K" is a constant with only the low-order 16 bits nonzero. 14177 if (isUInt<16>(Value)) 14178 Result = DAG.getTargetConstant(Value, dl, TCVT); 14179 break; 14180 case 'M': // "M" is a constant that is greater than 31. 14181 if (Value > 31) 14182 Result = DAG.getTargetConstant(Value, dl, TCVT); 14183 break; 14184 case 'N': // "N" is a positive constant that is an exact power of two. 14185 if (Value > 0 && isPowerOf2_64(Value)) 14186 Result = DAG.getTargetConstant(Value, dl, TCVT); 14187 break; 14188 case 'O': // "O" is the constant zero. 14189 if (Value == 0) 14190 Result = DAG.getTargetConstant(Value, dl, TCVT); 14191 break; 14192 case 'P': // "P" is a constant whose negation is a signed 16-bit constant. 14193 if (isInt<16>(-Value)) 14194 Result = DAG.getTargetConstant(Value, dl, TCVT); 14195 break; 14196 } 14197 break; 14198 } 14199 } 14200 14201 if (Result.getNode()) { 14202 Ops.push_back(Result); 14203 return; 14204 } 14205 14206 // Handle standard constraint letters. 14207 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 14208 } 14209 14210 // isLegalAddressingMode - Return true if the addressing mode represented 14211 // by AM is legal for this target, for a load/store of the specified type. 14212 bool PPCTargetLowering::isLegalAddressingMode(const DataLayout &DL, 14213 const AddrMode &AM, Type *Ty, 14214 unsigned AS, Instruction *I) const { 14215 // PPC does not allow r+i addressing modes for vectors! 14216 if (Ty->isVectorTy() && AM.BaseOffs != 0) 14217 return false; 14218 14219 // PPC allows a sign-extended 16-bit immediate field. 14220 if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1) 14221 return false; 14222 14223 // No global is ever allowed as a base. 14224 if (AM.BaseGV) 14225 return false; 14226 14227 // PPC only support r+r, 14228 switch (AM.Scale) { 14229 case 0: // "r+i" or just "i", depending on HasBaseReg. 14230 break; 14231 case 1: 14232 if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed. 14233 return false; 14234 // Otherwise we have r+r or r+i. 14235 break; 14236 case 2: 14237 if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed. 14238 return false; 14239 // Allow 2*r as r+r. 14240 break; 14241 default: 14242 // No other scales are supported. 14243 return false; 14244 } 14245 14246 return true; 14247 } 14248 14249 SDValue PPCTargetLowering::LowerRETURNADDR(SDValue Op, 14250 SelectionDAG &DAG) const { 14251 MachineFunction &MF = DAG.getMachineFunction(); 14252 MachineFrameInfo &MFI = MF.getFrameInfo(); 14253 MFI.setReturnAddressIsTaken(true); 14254 14255 if (verifyReturnAddressArgumentIsConstant(Op, DAG)) 14256 return SDValue(); 14257 14258 SDLoc dl(Op); 14259 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 14260 14261 // Make sure the function does not optimize away the store of the RA to 14262 // the stack. 14263 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 14264 FuncInfo->setLRStoreRequired(); 14265 bool isPPC64 = Subtarget.isPPC64(); 14266 auto PtrVT = getPointerTy(MF.getDataLayout()); 14267 14268 if (Depth > 0) { 14269 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG); 14270 SDValue Offset = 14271 DAG.getConstant(Subtarget.getFrameLowering()->getReturnSaveOffset(), dl, 14272 isPPC64 ? MVT::i64 : MVT::i32); 14273 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), 14274 DAG.getNode(ISD::ADD, dl, PtrVT, FrameAddr, Offset), 14275 MachinePointerInfo()); 14276 } 14277 14278 // Just load the return address off the stack. 14279 SDValue RetAddrFI = getReturnAddrFrameIndex(DAG); 14280 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), RetAddrFI, 14281 MachinePointerInfo()); 14282 } 14283 14284 SDValue PPCTargetLowering::LowerFRAMEADDR(SDValue Op, 14285 SelectionDAG &DAG) const { 14286 SDLoc dl(Op); 14287 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 14288 14289 MachineFunction &MF = DAG.getMachineFunction(); 14290 MachineFrameInfo &MFI = MF.getFrameInfo(); 14291 MFI.setFrameAddressIsTaken(true); 14292 14293 EVT PtrVT = getPointerTy(MF.getDataLayout()); 14294 bool isPPC64 = PtrVT == MVT::i64; 14295 14296 // Naked functions never have a frame pointer, and so we use r1. For all 14297 // other functions, this decision must be delayed until during PEI. 14298 unsigned FrameReg; 14299 if (MF.getFunction().hasFnAttribute(Attribute::Naked)) 14300 FrameReg = isPPC64 ? PPC::X1 : PPC::R1; 14301 else 14302 FrameReg = isPPC64 ? PPC::FP8 : PPC::FP; 14303 14304 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, 14305 PtrVT); 14306 while (Depth--) 14307 FrameAddr = DAG.getLoad(Op.getValueType(), dl, DAG.getEntryNode(), 14308 FrameAddr, MachinePointerInfo()); 14309 return FrameAddr; 14310 } 14311 14312 // FIXME? Maybe this could be a TableGen attribute on some registers and 14313 // this table could be generated automatically from RegInfo. 14314 unsigned PPCTargetLowering::getRegisterByName(const char* RegName, EVT VT, 14315 SelectionDAG &DAG) const { 14316 bool isPPC64 = Subtarget.isPPC64(); 14317 bool isDarwinABI = Subtarget.isDarwinABI(); 14318 14319 if ((isPPC64 && VT != MVT::i64 && VT != MVT::i32) || 14320 (!isPPC64 && VT != MVT::i32)) 14321 report_fatal_error("Invalid register global variable type"); 14322 14323 bool is64Bit = isPPC64 && VT == MVT::i64; 14324 unsigned Reg = StringSwitch<unsigned>(RegName) 14325 .Case("r1", is64Bit ? PPC::X1 : PPC::R1) 14326 .Case("r2", (isDarwinABI || isPPC64) ? 0 : PPC::R2) 14327 .Case("r13", (!isPPC64 && isDarwinABI) ? 0 : 14328 (is64Bit ? PPC::X13 : PPC::R13)) 14329 .Default(0); 14330 14331 if (Reg) 14332 return Reg; 14333 report_fatal_error("Invalid register name global variable"); 14334 } 14335 14336 bool PPCTargetLowering::isAccessedAsGotIndirect(SDValue GA) const { 14337 // 32-bit SVR4 ABI access everything as got-indirect. 14338 if (Subtarget.isSVR4ABI() && !Subtarget.isPPC64()) 14339 return true; 14340 14341 CodeModel::Model CModel = getTargetMachine().getCodeModel(); 14342 // If it is small or large code model, module locals are accessed 14343 // indirectly by loading their address from .toc/.got. The difference 14344 // is that for large code model we have ADDISTocHa + LDtocL and for 14345 // small code model we simply have LDtoc. 14346 if (CModel == CodeModel::Small || CModel == CodeModel::Large) 14347 return true; 14348 14349 // JumpTable and BlockAddress are accessed as got-indirect. 14350 if (isa<JumpTableSDNode>(GA) || isa<BlockAddressSDNode>(GA)) 14351 return true; 14352 14353 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(GA)) { 14354 const GlobalValue *GV = G->getGlobal(); 14355 unsigned char GVFlags = Subtarget.classifyGlobalReference(GV); 14356 // The NLP flag indicates that a global access has to use an 14357 // extra indirection. 14358 if (GVFlags & PPCII::MO_NLP_FLAG) 14359 return true; 14360 } 14361 14362 return false; 14363 } 14364 14365 bool 14366 PPCTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const { 14367 // The PowerPC target isn't yet aware of offsets. 14368 return false; 14369 } 14370 14371 bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, 14372 const CallInst &I, 14373 MachineFunction &MF, 14374 unsigned Intrinsic) const { 14375 switch (Intrinsic) { 14376 case Intrinsic::ppc_qpx_qvlfd: 14377 case Intrinsic::ppc_qpx_qvlfs: 14378 case Intrinsic::ppc_qpx_qvlfcd: 14379 case Intrinsic::ppc_qpx_qvlfcs: 14380 case Intrinsic::ppc_qpx_qvlfiwa: 14381 case Intrinsic::ppc_qpx_qvlfiwz: 14382 case Intrinsic::ppc_altivec_lvx: 14383 case Intrinsic::ppc_altivec_lvxl: 14384 case Intrinsic::ppc_altivec_lvebx: 14385 case Intrinsic::ppc_altivec_lvehx: 14386 case Intrinsic::ppc_altivec_lvewx: 14387 case Intrinsic::ppc_vsx_lxvd2x: 14388 case Intrinsic::ppc_vsx_lxvw4x: { 14389 EVT VT; 14390 switch (Intrinsic) { 14391 case Intrinsic::ppc_altivec_lvebx: 14392 VT = MVT::i8; 14393 break; 14394 case Intrinsic::ppc_altivec_lvehx: 14395 VT = MVT::i16; 14396 break; 14397 case Intrinsic::ppc_altivec_lvewx: 14398 VT = MVT::i32; 14399 break; 14400 case Intrinsic::ppc_vsx_lxvd2x: 14401 VT = MVT::v2f64; 14402 break; 14403 case Intrinsic::ppc_qpx_qvlfd: 14404 VT = MVT::v4f64; 14405 break; 14406 case Intrinsic::ppc_qpx_qvlfs: 14407 VT = MVT::v4f32; 14408 break; 14409 case Intrinsic::ppc_qpx_qvlfcd: 14410 VT = MVT::v2f64; 14411 break; 14412 case Intrinsic::ppc_qpx_qvlfcs: 14413 VT = MVT::v2f32; 14414 break; 14415 default: 14416 VT = MVT::v4i32; 14417 break; 14418 } 14419 14420 Info.opc = ISD::INTRINSIC_W_CHAIN; 14421 Info.memVT = VT; 14422 Info.ptrVal = I.getArgOperand(0); 14423 Info.offset = -VT.getStoreSize()+1; 14424 Info.size = 2*VT.getStoreSize()-1; 14425 Info.align = 1; 14426 Info.flags = MachineMemOperand::MOLoad; 14427 return true; 14428 } 14429 case Intrinsic::ppc_qpx_qvlfda: 14430 case Intrinsic::ppc_qpx_qvlfsa: 14431 case Intrinsic::ppc_qpx_qvlfcda: 14432 case Intrinsic::ppc_qpx_qvlfcsa: 14433 case Intrinsic::ppc_qpx_qvlfiwaa: 14434 case Intrinsic::ppc_qpx_qvlfiwza: { 14435 EVT VT; 14436 switch (Intrinsic) { 14437 case Intrinsic::ppc_qpx_qvlfda: 14438 VT = MVT::v4f64; 14439 break; 14440 case Intrinsic::ppc_qpx_qvlfsa: 14441 VT = MVT::v4f32; 14442 break; 14443 case Intrinsic::ppc_qpx_qvlfcda: 14444 VT = MVT::v2f64; 14445 break; 14446 case Intrinsic::ppc_qpx_qvlfcsa: 14447 VT = MVT::v2f32; 14448 break; 14449 default: 14450 VT = MVT::v4i32; 14451 break; 14452 } 14453 14454 Info.opc = ISD::INTRINSIC_W_CHAIN; 14455 Info.memVT = VT; 14456 Info.ptrVal = I.getArgOperand(0); 14457 Info.offset = 0; 14458 Info.size = VT.getStoreSize(); 14459 Info.align = 1; 14460 Info.flags = MachineMemOperand::MOLoad; 14461 return true; 14462 } 14463 case Intrinsic::ppc_qpx_qvstfd: 14464 case Intrinsic::ppc_qpx_qvstfs: 14465 case Intrinsic::ppc_qpx_qvstfcd: 14466 case Intrinsic::ppc_qpx_qvstfcs: 14467 case Intrinsic::ppc_qpx_qvstfiw: 14468 case Intrinsic::ppc_altivec_stvx: 14469 case Intrinsic::ppc_altivec_stvxl: 14470 case Intrinsic::ppc_altivec_stvebx: 14471 case Intrinsic::ppc_altivec_stvehx: 14472 case Intrinsic::ppc_altivec_stvewx: 14473 case Intrinsic::ppc_vsx_stxvd2x: 14474 case Intrinsic::ppc_vsx_stxvw4x: { 14475 EVT VT; 14476 switch (Intrinsic) { 14477 case Intrinsic::ppc_altivec_stvebx: 14478 VT = MVT::i8; 14479 break; 14480 case Intrinsic::ppc_altivec_stvehx: 14481 VT = MVT::i16; 14482 break; 14483 case Intrinsic::ppc_altivec_stvewx: 14484 VT = MVT::i32; 14485 break; 14486 case Intrinsic::ppc_vsx_stxvd2x: 14487 VT = MVT::v2f64; 14488 break; 14489 case Intrinsic::ppc_qpx_qvstfd: 14490 VT = MVT::v4f64; 14491 break; 14492 case Intrinsic::ppc_qpx_qvstfs: 14493 VT = MVT::v4f32; 14494 break; 14495 case Intrinsic::ppc_qpx_qvstfcd: 14496 VT = MVT::v2f64; 14497 break; 14498 case Intrinsic::ppc_qpx_qvstfcs: 14499 VT = MVT::v2f32; 14500 break; 14501 default: 14502 VT = MVT::v4i32; 14503 break; 14504 } 14505 14506 Info.opc = ISD::INTRINSIC_VOID; 14507 Info.memVT = VT; 14508 Info.ptrVal = I.getArgOperand(1); 14509 Info.offset = -VT.getStoreSize()+1; 14510 Info.size = 2*VT.getStoreSize()-1; 14511 Info.align = 1; 14512 Info.flags = MachineMemOperand::MOStore; 14513 return true; 14514 } 14515 case Intrinsic::ppc_qpx_qvstfda: 14516 case Intrinsic::ppc_qpx_qvstfsa: 14517 case Intrinsic::ppc_qpx_qvstfcda: 14518 case Intrinsic::ppc_qpx_qvstfcsa: 14519 case Intrinsic::ppc_qpx_qvstfiwa: { 14520 EVT VT; 14521 switch (Intrinsic) { 14522 case Intrinsic::ppc_qpx_qvstfda: 14523 VT = MVT::v4f64; 14524 break; 14525 case Intrinsic::ppc_qpx_qvstfsa: 14526 VT = MVT::v4f32; 14527 break; 14528 case Intrinsic::ppc_qpx_qvstfcda: 14529 VT = MVT::v2f64; 14530 break; 14531 case Intrinsic::ppc_qpx_qvstfcsa: 14532 VT = MVT::v2f32; 14533 break; 14534 default: 14535 VT = MVT::v4i32; 14536 break; 14537 } 14538 14539 Info.opc = ISD::INTRINSIC_VOID; 14540 Info.memVT = VT; 14541 Info.ptrVal = I.getArgOperand(1); 14542 Info.offset = 0; 14543 Info.size = VT.getStoreSize(); 14544 Info.align = 1; 14545 Info.flags = MachineMemOperand::MOStore; 14546 return true; 14547 } 14548 default: 14549 break; 14550 } 14551 14552 return false; 14553 } 14554 14555 /// getOptimalMemOpType - Returns the target specific optimal type for load 14556 /// and store operations as a result of memset, memcpy, and memmove 14557 /// lowering. If DstAlign is zero that means it's safe to destination 14558 /// alignment can satisfy any constraint. Similarly if SrcAlign is zero it 14559 /// means there isn't a need to check it against alignment requirement, 14560 /// probably because the source does not need to be loaded. If 'IsMemset' is 14561 /// true, that means it's expanding a memset. If 'ZeroMemset' is true, that 14562 /// means it's a memset of zero. 'MemcpyStrSrc' indicates whether the memcpy 14563 /// source is constant so it does not need to be loaded. 14564 /// It returns EVT::Other if the type should be determined using generic 14565 /// target-independent logic. 14566 EVT PPCTargetLowering::getOptimalMemOpType( 14567 uint64_t Size, unsigned DstAlign, unsigned SrcAlign, bool IsMemset, 14568 bool ZeroMemset, bool MemcpyStrSrc, 14569 const AttributeList &FuncAttributes) const { 14570 if (getTargetMachine().getOptLevel() != CodeGenOpt::None) { 14571 // When expanding a memset, require at least two QPX instructions to cover 14572 // the cost of loading the value to be stored from the constant pool. 14573 if (Subtarget.hasQPX() && Size >= 32 && (!IsMemset || Size >= 64) && 14574 (!SrcAlign || SrcAlign >= 32) && (!DstAlign || DstAlign >= 32) && 14575 !FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat)) { 14576 return MVT::v4f64; 14577 } 14578 14579 // We should use Altivec/VSX loads and stores when available. For unaligned 14580 // addresses, unaligned VSX loads are only fast starting with the P8. 14581 if (Subtarget.hasAltivec() && Size >= 16 && 14582 (((!SrcAlign || SrcAlign >= 16) && (!DstAlign || DstAlign >= 16)) || 14583 ((IsMemset && Subtarget.hasVSX()) || Subtarget.hasP8Vector()))) 14584 return MVT::v4i32; 14585 } 14586 14587 if (Subtarget.isPPC64()) { 14588 return MVT::i64; 14589 } 14590 14591 return MVT::i32; 14592 } 14593 14594 /// Returns true if it is beneficial to convert a load of a constant 14595 /// to just the constant itself. 14596 bool PPCTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm, 14597 Type *Ty) const { 14598 assert(Ty->isIntegerTy()); 14599 14600 unsigned BitSize = Ty->getPrimitiveSizeInBits(); 14601 return !(BitSize == 0 || BitSize > 64); 14602 } 14603 14604 bool PPCTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const { 14605 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy()) 14606 return false; 14607 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); 14608 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); 14609 return NumBits1 == 64 && NumBits2 == 32; 14610 } 14611 14612 bool PPCTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const { 14613 if (!VT1.isInteger() || !VT2.isInteger()) 14614 return false; 14615 unsigned NumBits1 = VT1.getSizeInBits(); 14616 unsigned NumBits2 = VT2.getSizeInBits(); 14617 return NumBits1 == 64 && NumBits2 == 32; 14618 } 14619 14620 bool PPCTargetLowering::isZExtFree(SDValue Val, EVT VT2) const { 14621 // Generally speaking, zexts are not free, but they are free when they can be 14622 // folded with other operations. 14623 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val)) { 14624 EVT MemVT = LD->getMemoryVT(); 14625 if ((MemVT == MVT::i1 || MemVT == MVT::i8 || MemVT == MVT::i16 || 14626 (Subtarget.isPPC64() && MemVT == MVT::i32)) && 14627 (LD->getExtensionType() == ISD::NON_EXTLOAD || 14628 LD->getExtensionType() == ISD::ZEXTLOAD)) 14629 return true; 14630 } 14631 14632 // FIXME: Add other cases... 14633 // - 32-bit shifts with a zext to i64 14634 // - zext after ctlz, bswap, etc. 14635 // - zext after and by a constant mask 14636 14637 return TargetLowering::isZExtFree(Val, VT2); 14638 } 14639 14640 bool PPCTargetLowering::isFPExtFree(EVT DestVT, EVT SrcVT) const { 14641 assert(DestVT.isFloatingPoint() && SrcVT.isFloatingPoint() && 14642 "invalid fpext types"); 14643 // Extending to float128 is not free. 14644 if (DestVT == MVT::f128) 14645 return false; 14646 return true; 14647 } 14648 14649 bool PPCTargetLowering::isLegalICmpImmediate(int64_t Imm) const { 14650 return isInt<16>(Imm) || isUInt<16>(Imm); 14651 } 14652 14653 bool PPCTargetLowering::isLegalAddImmediate(int64_t Imm) const { 14654 return isInt<16>(Imm) || isUInt<16>(Imm); 14655 } 14656 14657 bool PPCTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, 14658 unsigned, 14659 unsigned, 14660 MachineMemOperand::Flags, 14661 bool *Fast) const { 14662 if (DisablePPCUnaligned) 14663 return false; 14664 14665 // PowerPC supports unaligned memory access for simple non-vector types. 14666 // Although accessing unaligned addresses is not as efficient as accessing 14667 // aligned addresses, it is generally more efficient than manual expansion, 14668 // and generally only traps for software emulation when crossing page 14669 // boundaries. 14670 14671 if (!VT.isSimple()) 14672 return false; 14673 14674 if (VT.getSimpleVT().isVector()) { 14675 if (Subtarget.hasVSX()) { 14676 if (VT != MVT::v2f64 && VT != MVT::v2i64 && 14677 VT != MVT::v4f32 && VT != MVT::v4i32) 14678 return false; 14679 } else { 14680 return false; 14681 } 14682 } 14683 14684 if (VT == MVT::ppcf128) 14685 return false; 14686 14687 if (Fast) 14688 *Fast = true; 14689 14690 return true; 14691 } 14692 14693 bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const { 14694 VT = VT.getScalarType(); 14695 14696 if (!VT.isSimple()) 14697 return false; 14698 14699 switch (VT.getSimpleVT().SimpleTy) { 14700 case MVT::f32: 14701 case MVT::f64: 14702 return true; 14703 case MVT::f128: 14704 return (EnableQuadPrecision && Subtarget.hasP9Vector()); 14705 default: 14706 break; 14707 } 14708 14709 return false; 14710 } 14711 14712 const MCPhysReg * 14713 PPCTargetLowering::getScratchRegisters(CallingConv::ID) const { 14714 // LR is a callee-save register, but we must treat it as clobbered by any call 14715 // site. Hence we include LR in the scratch registers, which are in turn added 14716 // as implicit-defs for stackmaps and patchpoints. The same reasoning applies 14717 // to CTR, which is used by any indirect call. 14718 static const MCPhysReg ScratchRegs[] = { 14719 PPC::X12, PPC::LR8, PPC::CTR8, 0 14720 }; 14721 14722 return ScratchRegs; 14723 } 14724 14725 unsigned PPCTargetLowering::getExceptionPointerRegister( 14726 const Constant *PersonalityFn) const { 14727 return Subtarget.isPPC64() ? PPC::X3 : PPC::R3; 14728 } 14729 14730 unsigned PPCTargetLowering::getExceptionSelectorRegister( 14731 const Constant *PersonalityFn) const { 14732 return Subtarget.isPPC64() ? PPC::X4 : PPC::R4; 14733 } 14734 14735 bool 14736 PPCTargetLowering::shouldExpandBuildVectorWithShuffles( 14737 EVT VT , unsigned DefinedValues) const { 14738 if (VT == MVT::v2i64) 14739 return Subtarget.hasDirectMove(); // Don't need stack ops with direct moves 14740 14741 if (Subtarget.hasVSX() || Subtarget.hasQPX()) 14742 return true; 14743 14744 return TargetLowering::shouldExpandBuildVectorWithShuffles(VT, DefinedValues); 14745 } 14746 14747 Sched::Preference PPCTargetLowering::getSchedulingPreference(SDNode *N) const { 14748 if (DisableILPPref || Subtarget.enableMachineScheduler()) 14749 return TargetLowering::getSchedulingPreference(N); 14750 14751 return Sched::ILP; 14752 } 14753 14754 // Create a fast isel object. 14755 FastISel * 14756 PPCTargetLowering::createFastISel(FunctionLoweringInfo &FuncInfo, 14757 const TargetLibraryInfo *LibInfo) const { 14758 return PPC::createFastISel(FuncInfo, LibInfo); 14759 } 14760 14761 void PPCTargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const { 14762 if (Subtarget.isDarwinABI()) return; 14763 if (!Subtarget.isPPC64()) return; 14764 14765 // Update IsSplitCSR in PPCFunctionInfo 14766 PPCFunctionInfo *PFI = Entry->getParent()->getInfo<PPCFunctionInfo>(); 14767 PFI->setIsSplitCSR(true); 14768 } 14769 14770 void PPCTargetLowering::insertCopiesSplitCSR( 14771 MachineBasicBlock *Entry, 14772 const SmallVectorImpl<MachineBasicBlock *> &Exits) const { 14773 const PPCRegisterInfo *TRI = Subtarget.getRegisterInfo(); 14774 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent()); 14775 if (!IStart) 14776 return; 14777 14778 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 14779 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo(); 14780 MachineBasicBlock::iterator MBBI = Entry->begin(); 14781 for (const MCPhysReg *I = IStart; *I; ++I) { 14782 const TargetRegisterClass *RC = nullptr; 14783 if (PPC::G8RCRegClass.contains(*I)) 14784 RC = &PPC::G8RCRegClass; 14785 else if (PPC::F8RCRegClass.contains(*I)) 14786 RC = &PPC::F8RCRegClass; 14787 else if (PPC::CRRCRegClass.contains(*I)) 14788 RC = &PPC::CRRCRegClass; 14789 else if (PPC::VRRCRegClass.contains(*I)) 14790 RC = &PPC::VRRCRegClass; 14791 else 14792 llvm_unreachable("Unexpected register class in CSRsViaCopy!"); 14793 14794 unsigned NewVR = MRI->createVirtualRegister(RC); 14795 // Create copy from CSR to a virtual register. 14796 // FIXME: this currently does not emit CFI pseudo-instructions, it works 14797 // fine for CXX_FAST_TLS since the C++-style TLS access functions should be 14798 // nounwind. If we want to generalize this later, we may need to emit 14799 // CFI pseudo-instructions. 14800 assert(Entry->getParent()->getFunction().hasFnAttribute( 14801 Attribute::NoUnwind) && 14802 "Function should be nounwind in insertCopiesSplitCSR!"); 14803 Entry->addLiveIn(*I); 14804 BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR) 14805 .addReg(*I); 14806 14807 // Insert the copy-back instructions right before the terminator. 14808 for (auto *Exit : Exits) 14809 BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(), 14810 TII->get(TargetOpcode::COPY), *I) 14811 .addReg(NewVR); 14812 } 14813 } 14814 14815 // Override to enable LOAD_STACK_GUARD lowering on Linux. 14816 bool PPCTargetLowering::useLoadStackGuardNode() const { 14817 if (!Subtarget.isTargetLinux()) 14818 return TargetLowering::useLoadStackGuardNode(); 14819 return true; 14820 } 14821 14822 // Override to disable global variable loading on Linux. 14823 void PPCTargetLowering::insertSSPDeclarations(Module &M) const { 14824 if (!Subtarget.isTargetLinux()) 14825 return TargetLowering::insertSSPDeclarations(M); 14826 } 14827 14828 bool PPCTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, 14829 bool ForCodeSize) const { 14830 if (!VT.isSimple() || !Subtarget.hasVSX()) 14831 return false; 14832 14833 switch(VT.getSimpleVT().SimpleTy) { 14834 default: 14835 // For FP types that are currently not supported by PPC backend, return 14836 // false. Examples: f16, f80. 14837 return false; 14838 case MVT::f32: 14839 case MVT::f64: 14840 case MVT::ppcf128: 14841 return Imm.isPosZero(); 14842 } 14843 } 14844 14845 // For vector shift operation op, fold 14846 // (op x, (and y, ((1 << numbits(x)) - 1))) -> (target op x, y) 14847 static SDValue stripModuloOnShift(const TargetLowering &TLI, SDNode *N, 14848 SelectionDAG &DAG) { 14849 SDValue N0 = N->getOperand(0); 14850 SDValue N1 = N->getOperand(1); 14851 EVT VT = N0.getValueType(); 14852 unsigned OpSizeInBits = VT.getScalarSizeInBits(); 14853 unsigned Opcode = N->getOpcode(); 14854 unsigned TargetOpcode; 14855 14856 switch (Opcode) { 14857 default: 14858 llvm_unreachable("Unexpected shift operation"); 14859 case ISD::SHL: 14860 TargetOpcode = PPCISD::SHL; 14861 break; 14862 case ISD::SRL: 14863 TargetOpcode = PPCISD::SRL; 14864 break; 14865 case ISD::SRA: 14866 TargetOpcode = PPCISD::SRA; 14867 break; 14868 } 14869 14870 if (VT.isVector() && TLI.isOperationLegal(Opcode, VT) && 14871 N1->getOpcode() == ISD::AND) 14872 if (ConstantSDNode *Mask = isConstOrConstSplat(N1->getOperand(1))) 14873 if (Mask->getZExtValue() == OpSizeInBits - 1) 14874 return DAG.getNode(TargetOpcode, SDLoc(N), VT, N0, N1->getOperand(0)); 14875 14876 return SDValue(); 14877 } 14878 14879 SDValue PPCTargetLowering::combineSHL(SDNode *N, DAGCombinerInfo &DCI) const { 14880 if (auto Value = stripModuloOnShift(*this, N, DCI.DAG)) 14881 return Value; 14882 14883 SDValue N0 = N->getOperand(0); 14884 ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(N->getOperand(1)); 14885 if (!Subtarget.isISA3_0() || 14886 N0.getOpcode() != ISD::SIGN_EXTEND || 14887 N0.getOperand(0).getValueType() != MVT::i32 || 14888 CN1 == nullptr || N->getValueType(0) != MVT::i64) 14889 return SDValue(); 14890 14891 // We can't save an operation here if the value is already extended, and 14892 // the existing shift is easier to combine. 14893 SDValue ExtsSrc = N0.getOperand(0); 14894 if (ExtsSrc.getOpcode() == ISD::TRUNCATE && 14895 ExtsSrc.getOperand(0).getOpcode() == ISD::AssertSext) 14896 return SDValue(); 14897 14898 SDLoc DL(N0); 14899 SDValue ShiftBy = SDValue(CN1, 0); 14900 // We want the shift amount to be i32 on the extswli, but the shift could 14901 // have an i64. 14902 if (ShiftBy.getValueType() == MVT::i64) 14903 ShiftBy = DCI.DAG.getConstant(CN1->getZExtValue(), DL, MVT::i32); 14904 14905 return DCI.DAG.getNode(PPCISD::EXTSWSLI, DL, MVT::i64, N0->getOperand(0), 14906 ShiftBy); 14907 } 14908 14909 SDValue PPCTargetLowering::combineSRA(SDNode *N, DAGCombinerInfo &DCI) const { 14910 if (auto Value = stripModuloOnShift(*this, N, DCI.DAG)) 14911 return Value; 14912 14913 return SDValue(); 14914 } 14915 14916 SDValue PPCTargetLowering::combineSRL(SDNode *N, DAGCombinerInfo &DCI) const { 14917 if (auto Value = stripModuloOnShift(*this, N, DCI.DAG)) 14918 return Value; 14919 14920 return SDValue(); 14921 } 14922 14923 // Transform (add X, (zext(setne Z, C))) -> (addze X, (addic (addi Z, -C), -1)) 14924 // Transform (add X, (zext(sete Z, C))) -> (addze X, (subfic (addi Z, -C), 0)) 14925 // When C is zero, the equation (addi Z, -C) can be simplified to Z 14926 // Requirement: -C in [-32768, 32767], X and Z are MVT::i64 types 14927 static SDValue combineADDToADDZE(SDNode *N, SelectionDAG &DAG, 14928 const PPCSubtarget &Subtarget) { 14929 if (!Subtarget.isPPC64()) 14930 return SDValue(); 14931 14932 SDValue LHS = N->getOperand(0); 14933 SDValue RHS = N->getOperand(1); 14934 14935 auto isZextOfCompareWithConstant = [](SDValue Op) { 14936 if (Op.getOpcode() != ISD::ZERO_EXTEND || !Op.hasOneUse() || 14937 Op.getValueType() != MVT::i64) 14938 return false; 14939 14940 SDValue Cmp = Op.getOperand(0); 14941 if (Cmp.getOpcode() != ISD::SETCC || !Cmp.hasOneUse() || 14942 Cmp.getOperand(0).getValueType() != MVT::i64) 14943 return false; 14944 14945 if (auto *Constant = dyn_cast<ConstantSDNode>(Cmp.getOperand(1))) { 14946 int64_t NegConstant = 0 - Constant->getSExtValue(); 14947 // Due to the limitations of the addi instruction, 14948 // -C is required to be [-32768, 32767]. 14949 return isInt<16>(NegConstant); 14950 } 14951 14952 return false; 14953 }; 14954 14955 bool LHSHasPattern = isZextOfCompareWithConstant(LHS); 14956 bool RHSHasPattern = isZextOfCompareWithConstant(RHS); 14957 14958 // If there is a pattern, canonicalize a zext operand to the RHS. 14959 if (LHSHasPattern && !RHSHasPattern) 14960 std::swap(LHS, RHS); 14961 else if (!LHSHasPattern && !RHSHasPattern) 14962 return SDValue(); 14963 14964 SDLoc DL(N); 14965 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Glue); 14966 SDValue Cmp = RHS.getOperand(0); 14967 SDValue Z = Cmp.getOperand(0); 14968 auto *Constant = dyn_cast<ConstantSDNode>(Cmp.getOperand(1)); 14969 14970 assert(Constant && "Constant Should not be a null pointer."); 14971 int64_t NegConstant = 0 - Constant->getSExtValue(); 14972 14973 switch(cast<CondCodeSDNode>(Cmp.getOperand(2))->get()) { 14974 default: break; 14975 case ISD::SETNE: { 14976 // when C == 0 14977 // --> addze X, (addic Z, -1).carry 14978 // / 14979 // add X, (zext(setne Z, C))-- 14980 // \ when -32768 <= -C <= 32767 && C != 0 14981 // --> addze X, (addic (addi Z, -C), -1).carry 14982 SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Z, 14983 DAG.getConstant(NegConstant, DL, MVT::i64)); 14984 SDValue AddOrZ = NegConstant != 0 ? Add : Z; 14985 SDValue Addc = DAG.getNode(ISD::ADDC, DL, DAG.getVTList(MVT::i64, MVT::Glue), 14986 AddOrZ, DAG.getConstant(-1ULL, DL, MVT::i64)); 14987 return DAG.getNode(ISD::ADDE, DL, VTs, LHS, DAG.getConstant(0, DL, MVT::i64), 14988 SDValue(Addc.getNode(), 1)); 14989 } 14990 case ISD::SETEQ: { 14991 // when C == 0 14992 // --> addze X, (subfic Z, 0).carry 14993 // / 14994 // add X, (zext(sete Z, C))-- 14995 // \ when -32768 <= -C <= 32767 && C != 0 14996 // --> addze X, (subfic (addi Z, -C), 0).carry 14997 SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Z, 14998 DAG.getConstant(NegConstant, DL, MVT::i64)); 14999 SDValue AddOrZ = NegConstant != 0 ? Add : Z; 15000 SDValue Subc = DAG.getNode(ISD::SUBC, DL, DAG.getVTList(MVT::i64, MVT::Glue), 15001 DAG.getConstant(0, DL, MVT::i64), AddOrZ); 15002 return DAG.getNode(ISD::ADDE, DL, VTs, LHS, DAG.getConstant(0, DL, MVT::i64), 15003 SDValue(Subc.getNode(), 1)); 15004 } 15005 } 15006 15007 return SDValue(); 15008 } 15009 15010 SDValue PPCTargetLowering::combineADD(SDNode *N, DAGCombinerInfo &DCI) const { 15011 if (auto Value = combineADDToADDZE(N, DCI.DAG, Subtarget)) 15012 return Value; 15013 15014 return SDValue(); 15015 } 15016 15017 // Detect TRUNCATE operations on bitcasts of float128 values. 15018 // What we are looking for here is the situtation where we extract a subset 15019 // of bits from a 128 bit float. 15020 // This can be of two forms: 15021 // 1) BITCAST of f128 feeding TRUNCATE 15022 // 2) BITCAST of f128 feeding SRL (a shift) feeding TRUNCATE 15023 // The reason this is required is because we do not have a legal i128 type 15024 // and so we want to prevent having to store the f128 and then reload part 15025 // of it. 15026 SDValue PPCTargetLowering::combineTRUNCATE(SDNode *N, 15027 DAGCombinerInfo &DCI) const { 15028 // If we are using CRBits then try that first. 15029 if (Subtarget.useCRBits()) { 15030 // Check if CRBits did anything and return that if it did. 15031 if (SDValue CRTruncValue = DAGCombineTruncBoolExt(N, DCI)) 15032 return CRTruncValue; 15033 } 15034 15035 SDLoc dl(N); 15036 SDValue Op0 = N->getOperand(0); 15037 15038 // Looking for a truncate of i128 to i64. 15039 if (Op0.getValueType() != MVT::i128 || N->getValueType(0) != MVT::i64) 15040 return SDValue(); 15041 15042 int EltToExtract = DCI.DAG.getDataLayout().isBigEndian() ? 1 : 0; 15043 15044 // SRL feeding TRUNCATE. 15045 if (Op0.getOpcode() == ISD::SRL) { 15046 ConstantSDNode *ConstNode = dyn_cast<ConstantSDNode>(Op0.getOperand(1)); 15047 // The right shift has to be by 64 bits. 15048 if (!ConstNode || ConstNode->getZExtValue() != 64) 15049 return SDValue(); 15050 15051 // Switch the element number to extract. 15052 EltToExtract = EltToExtract ? 0 : 1; 15053 // Update Op0 past the SRL. 15054 Op0 = Op0.getOperand(0); 15055 } 15056 15057 // BITCAST feeding a TRUNCATE possibly via SRL. 15058 if (Op0.getOpcode() == ISD::BITCAST && 15059 Op0.getValueType() == MVT::i128 && 15060 Op0.getOperand(0).getValueType() == MVT::f128) { 15061 SDValue Bitcast = DCI.DAG.getBitcast(MVT::v2i64, Op0.getOperand(0)); 15062 return DCI.DAG.getNode( 15063 ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Bitcast, 15064 DCI.DAG.getTargetConstant(EltToExtract, dl, MVT::i32)); 15065 } 15066 return SDValue(); 15067 } 15068 15069 SDValue PPCTargetLowering::combineMUL(SDNode *N, DAGCombinerInfo &DCI) const { 15070 SelectionDAG &DAG = DCI.DAG; 15071 15072 ConstantSDNode *ConstOpOrElement = isConstOrConstSplat(N->getOperand(1)); 15073 if (!ConstOpOrElement) 15074 return SDValue(); 15075 15076 // An imul is usually smaller than the alternative sequence for legal type. 15077 if (DAG.getMachineFunction().getFunction().hasMinSize() && 15078 isOperationLegal(ISD::MUL, N->getValueType(0))) 15079 return SDValue(); 15080 15081 auto IsProfitable = [this](bool IsNeg, bool IsAddOne, EVT VT) -> bool { 15082 switch (this->Subtarget.getDarwinDirective()) { 15083 default: 15084 // TODO: enhance the condition for subtarget before pwr8 15085 return false; 15086 case PPC::DIR_PWR8: 15087 // type mul add shl 15088 // scalar 4 1 1 15089 // vector 7 2 2 15090 return true; 15091 case PPC::DIR_PWR9: 15092 // type mul add shl 15093 // scalar 5 2 2 15094 // vector 7 2 2 15095 15096 // The cycle RATIO of related operations are showed as a table above. 15097 // Because mul is 5(scalar)/7(vector), add/sub/shl are all 2 for both 15098 // scalar and vector type. For 2 instrs patterns, add/sub + shl 15099 // are 4, it is always profitable; but for 3 instrs patterns 15100 // (mul x, -(2^N + 1)) => -(add (shl x, N), x), sub + add + shl are 6. 15101 // So we should only do it for vector type. 15102 return IsAddOne && IsNeg ? VT.isVector() : true; 15103 } 15104 }; 15105 15106 EVT VT = N->getValueType(0); 15107 SDLoc DL(N); 15108 15109 const APInt &MulAmt = ConstOpOrElement->getAPIntValue(); 15110 bool IsNeg = MulAmt.isNegative(); 15111 APInt MulAmtAbs = MulAmt.abs(); 15112 15113 if ((MulAmtAbs - 1).isPowerOf2()) { 15114 // (mul x, 2^N + 1) => (add (shl x, N), x) 15115 // (mul x, -(2^N + 1)) => -(add (shl x, N), x) 15116 15117 if (!IsProfitable(IsNeg, true, VT)) 15118 return SDValue(); 15119 15120 SDValue Op0 = N->getOperand(0); 15121 SDValue Op1 = 15122 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), 15123 DAG.getConstant((MulAmtAbs - 1).logBase2(), DL, VT)); 15124 SDValue Res = DAG.getNode(ISD::ADD, DL, VT, Op0, Op1); 15125 15126 if (!IsNeg) 15127 return Res; 15128 15129 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Res); 15130 } else if ((MulAmtAbs + 1).isPowerOf2()) { 15131 // (mul x, 2^N - 1) => (sub (shl x, N), x) 15132 // (mul x, -(2^N - 1)) => (sub x, (shl x, N)) 15133 15134 if (!IsProfitable(IsNeg, false, VT)) 15135 return SDValue(); 15136 15137 SDValue Op0 = N->getOperand(0); 15138 SDValue Op1 = 15139 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), 15140 DAG.getConstant((MulAmtAbs + 1).logBase2(), DL, VT)); 15141 15142 if (!IsNeg) 15143 return DAG.getNode(ISD::SUB, DL, VT, Op1, Op0); 15144 else 15145 return DAG.getNode(ISD::SUB, DL, VT, Op0, Op1); 15146 15147 } else { 15148 return SDValue(); 15149 } 15150 } 15151 15152 bool PPCTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const { 15153 // Only duplicate to increase tail-calls for the 64bit SysV ABIs. 15154 if (!Subtarget.isSVR4ABI() || !Subtarget.isPPC64()) 15155 return false; 15156 15157 // If not a tail call then no need to proceed. 15158 if (!CI->isTailCall()) 15159 return false; 15160 15161 // If tail calls are disabled for the caller then we are done. 15162 const Function *Caller = CI->getParent()->getParent(); 15163 auto Attr = Caller->getFnAttribute("disable-tail-calls"); 15164 if (Attr.getValueAsString() == "true") 15165 return false; 15166 15167 // If sibling calls have been disabled and tail-calls aren't guaranteed 15168 // there is no reason to duplicate. 15169 auto &TM = getTargetMachine(); 15170 if (!TM.Options.GuaranteedTailCallOpt && DisableSCO) 15171 return false; 15172 15173 // Can't tail call a function called indirectly, or if it has variadic args. 15174 const Function *Callee = CI->getCalledFunction(); 15175 if (!Callee || Callee->isVarArg()) 15176 return false; 15177 15178 // Make sure the callee and caller calling conventions are eligible for tco. 15179 if (!areCallingConvEligibleForTCO_64SVR4(Caller->getCallingConv(), 15180 CI->getCallingConv())) 15181 return false; 15182 15183 // If the function is local then we have a good chance at tail-calling it 15184 return getTargetMachine().shouldAssumeDSOLocal(*Caller->getParent(), Callee); 15185 } 15186 15187 bool PPCTargetLowering::hasBitPreservingFPLogic(EVT VT) const { 15188 if (!Subtarget.hasVSX()) 15189 return false; 15190 if (Subtarget.hasP9Vector() && VT == MVT::f128) 15191 return true; 15192 return VT == MVT::f32 || VT == MVT::f64 || 15193 VT == MVT::v4f32 || VT == MVT::v2f64; 15194 } 15195 15196 bool PPCTargetLowering:: 15197 isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const { 15198 const Value *Mask = AndI.getOperand(1); 15199 // If the mask is suitable for andi. or andis. we should sink the and. 15200 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Mask)) { 15201 // Can't handle constants wider than 64-bits. 15202 if (CI->getBitWidth() > 64) 15203 return false; 15204 int64_t ConstVal = CI->getZExtValue(); 15205 return isUInt<16>(ConstVal) || 15206 (isUInt<16>(ConstVal >> 16) && !(ConstVal & 0xFFFF)); 15207 } 15208 15209 // For non-constant masks, we can always use the record-form and. 15210 return true; 15211 } 15212 15213 // Transform (abs (sub (zext a), (zext b))) to (vabsd a b 0) 15214 // Transform (abs (sub (zext a), (zext_invec b))) to (vabsd a b 0) 15215 // Transform (abs (sub (zext_invec a), (zext_invec b))) to (vabsd a b 0) 15216 // Transform (abs (sub (zext_invec a), (zext b))) to (vabsd a b 0) 15217 // Transform (abs (sub a, b) to (vabsd a b 1)) if a & b of type v4i32 15218 SDValue PPCTargetLowering::combineABS(SDNode *N, DAGCombinerInfo &DCI) const { 15219 assert((N->getOpcode() == ISD::ABS) && "Need ABS node here"); 15220 assert(Subtarget.hasP9Altivec() && 15221 "Only combine this when P9 altivec supported!"); 15222 EVT VT = N->getValueType(0); 15223 if (VT != MVT::v4i32 && VT != MVT::v8i16 && VT != MVT::v16i8) 15224 return SDValue(); 15225 15226 SelectionDAG &DAG = DCI.DAG; 15227 SDLoc dl(N); 15228 if (N->getOperand(0).getOpcode() == ISD::SUB) { 15229 // Even for signed integers, if it's known to be positive (as signed 15230 // integer) due to zero-extended inputs. 15231 unsigned SubOpcd0 = N->getOperand(0)->getOperand(0).getOpcode(); 15232 unsigned SubOpcd1 = N->getOperand(0)->getOperand(1).getOpcode(); 15233 if ((SubOpcd0 == ISD::ZERO_EXTEND || 15234 SubOpcd0 == ISD::ZERO_EXTEND_VECTOR_INREG) && 15235 (SubOpcd1 == ISD::ZERO_EXTEND || 15236 SubOpcd1 == ISD::ZERO_EXTEND_VECTOR_INREG)) { 15237 return DAG.getNode(PPCISD::VABSD, dl, N->getOperand(0).getValueType(), 15238 N->getOperand(0)->getOperand(0), 15239 N->getOperand(0)->getOperand(1), 15240 DAG.getTargetConstant(0, dl, MVT::i32)); 15241 } 15242 15243 // For type v4i32, it can be optimized with xvnegsp + vabsduw 15244 if (N->getOperand(0).getValueType() == MVT::v4i32 && 15245 N->getOperand(0).hasOneUse()) { 15246 return DAG.getNode(PPCISD::VABSD, dl, N->getOperand(0).getValueType(), 15247 N->getOperand(0)->getOperand(0), 15248 N->getOperand(0)->getOperand(1), 15249 DAG.getTargetConstant(1, dl, MVT::i32)); 15250 } 15251 } 15252 15253 return SDValue(); 15254 } 15255 15256 // For type v4i32/v8ii16/v16i8, transform 15257 // from (vselect (setcc a, b, setugt), (sub a, b), (sub b, a)) to (vabsd a, b) 15258 // from (vselect (setcc a, b, setuge), (sub a, b), (sub b, a)) to (vabsd a, b) 15259 // from (vselect (setcc a, b, setult), (sub b, a), (sub a, b)) to (vabsd a, b) 15260 // from (vselect (setcc a, b, setule), (sub b, a), (sub a, b)) to (vabsd a, b) 15261 SDValue PPCTargetLowering::combineVSelect(SDNode *N, 15262 DAGCombinerInfo &DCI) const { 15263 assert((N->getOpcode() == ISD::VSELECT) && "Need VSELECT node here"); 15264 assert(Subtarget.hasP9Altivec() && 15265 "Only combine this when P9 altivec supported!"); 15266 15267 SelectionDAG &DAG = DCI.DAG; 15268 SDLoc dl(N); 15269 SDValue Cond = N->getOperand(0); 15270 SDValue TrueOpnd = N->getOperand(1); 15271 SDValue FalseOpnd = N->getOperand(2); 15272 EVT VT = N->getOperand(1).getValueType(); 15273 15274 if (Cond.getOpcode() != ISD::SETCC || TrueOpnd.getOpcode() != ISD::SUB || 15275 FalseOpnd.getOpcode() != ISD::SUB) 15276 return SDValue(); 15277 15278 // ABSD only available for type v4i32/v8i16/v16i8 15279 if (VT != MVT::v4i32 && VT != MVT::v8i16 && VT != MVT::v16i8) 15280 return SDValue(); 15281 15282 // At least to save one more dependent computation 15283 if (!(Cond.hasOneUse() || TrueOpnd.hasOneUse() || FalseOpnd.hasOneUse())) 15284 return SDValue(); 15285 15286 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); 15287 15288 // Can only handle unsigned comparison here 15289 switch (CC) { 15290 default: 15291 return SDValue(); 15292 case ISD::SETUGT: 15293 case ISD::SETUGE: 15294 break; 15295 case ISD::SETULT: 15296 case ISD::SETULE: 15297 std::swap(TrueOpnd, FalseOpnd); 15298 break; 15299 } 15300 15301 SDValue CmpOpnd1 = Cond.getOperand(0); 15302 SDValue CmpOpnd2 = Cond.getOperand(1); 15303 15304 // SETCC CmpOpnd1 CmpOpnd2 cond 15305 // TrueOpnd = CmpOpnd1 - CmpOpnd2 15306 // FalseOpnd = CmpOpnd2 - CmpOpnd1 15307 if (TrueOpnd.getOperand(0) == CmpOpnd1 && 15308 TrueOpnd.getOperand(1) == CmpOpnd2 && 15309 FalseOpnd.getOperand(0) == CmpOpnd2 && 15310 FalseOpnd.getOperand(1) == CmpOpnd1) { 15311 return DAG.getNode(PPCISD::VABSD, dl, N->getOperand(1).getValueType(), 15312 CmpOpnd1, CmpOpnd2, 15313 DAG.getTargetConstant(0, dl, MVT::i32)); 15314 } 15315 15316 return SDValue(); 15317 } 15318