1 //===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the SelectionDAG::Legalize method. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/ADT/APFloat.h" 14 #include "llvm/ADT/APInt.h" 15 #include "llvm/ADT/ArrayRef.h" 16 #include "llvm/ADT/SetVector.h" 17 #include "llvm/ADT/SmallPtrSet.h" 18 #include "llvm/ADT/SmallSet.h" 19 #include "llvm/ADT/SmallVector.h" 20 #include "llvm/Analysis/TargetLibraryInfo.h" 21 #include "llvm/CodeGen/ISDOpcodes.h" 22 #include "llvm/CodeGen/MachineFunction.h" 23 #include "llvm/CodeGen/MachineJumpTableInfo.h" 24 #include "llvm/CodeGen/MachineMemOperand.h" 25 #include "llvm/CodeGen/RuntimeLibcalls.h" 26 #include "llvm/CodeGen/SelectionDAG.h" 27 #include "llvm/CodeGen/SelectionDAGNodes.h" 28 #include "llvm/CodeGen/TargetFrameLowering.h" 29 #include "llvm/CodeGen/TargetLowering.h" 30 #include "llvm/CodeGen/TargetSubtargetInfo.h" 31 #include "llvm/CodeGen/ValueTypes.h" 32 #include "llvm/IR/CallingConv.h" 33 #include "llvm/IR/Constants.h" 34 #include "llvm/IR/DataLayout.h" 35 #include "llvm/IR/DerivedTypes.h" 36 #include "llvm/IR/Function.h" 37 #include "llvm/IR/Metadata.h" 38 #include "llvm/IR/Type.h" 39 #include "llvm/Support/Casting.h" 40 #include "llvm/Support/Compiler.h" 41 #include "llvm/Support/Debug.h" 42 #include "llvm/Support/ErrorHandling.h" 43 #include "llvm/Support/MachineValueType.h" 44 #include "llvm/Support/MathExtras.h" 45 #include "llvm/Support/raw_ostream.h" 46 #include "llvm/Target/TargetMachine.h" 47 #include "llvm/Target/TargetOptions.h" 48 #include <algorithm> 49 #include <cassert> 50 #include <cstdint> 51 #include <tuple> 52 #include <utility> 53 54 using namespace llvm; 55 56 #define DEBUG_TYPE "legalizedag" 57 58 namespace { 59 60 /// Keeps track of state when getting the sign of a floating-point value as an 61 /// integer. 62 struct FloatSignAsInt { 63 EVT FloatVT; 64 SDValue Chain; 65 SDValue FloatPtr; 66 SDValue IntPtr; 67 MachinePointerInfo IntPointerInfo; 68 MachinePointerInfo FloatPointerInfo; 69 SDValue IntValue; 70 APInt SignMask; 71 uint8_t SignBit; 72 }; 73 74 //===----------------------------------------------------------------------===// 75 /// This takes an arbitrary SelectionDAG as input and 76 /// hacks on it until the target machine can handle it. This involves 77 /// eliminating value sizes the machine cannot handle (promoting small sizes to 78 /// large sizes or splitting up large values into small values) as well as 79 /// eliminating operations the machine cannot handle. 80 /// 81 /// This code also does a small amount of optimization and recognition of idioms 82 /// as part of its processing. For example, if a target does not support a 83 /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this 84 /// will attempt merge setcc and brc instructions into brcc's. 85 class SelectionDAGLegalize { 86 const TargetMachine &TM; 87 const TargetLowering &TLI; 88 SelectionDAG &DAG; 89 90 /// The set of nodes which have already been legalized. We hold a 91 /// reference to it in order to update as necessary on node deletion. 92 SmallPtrSetImpl<SDNode *> &LegalizedNodes; 93 94 /// A set of all the nodes updated during legalization. 95 SmallSetVector<SDNode *, 16> *UpdatedNodes; 96 97 EVT getSetCCResultType(EVT VT) const { 98 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); 99 } 100 101 // Libcall insertion helpers. 102 103 public: 104 SelectionDAGLegalize(SelectionDAG &DAG, 105 SmallPtrSetImpl<SDNode *> &LegalizedNodes, 106 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr) 107 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG), 108 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {} 109 110 /// Legalizes the given operation. 111 void LegalizeOp(SDNode *Node); 112 113 private: 114 SDValue OptimizeFloatStore(StoreSDNode *ST); 115 116 void LegalizeLoadOps(SDNode *Node); 117 void LegalizeStoreOps(SDNode *Node); 118 119 /// Some targets cannot handle a variable 120 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it 121 /// is necessary to spill the vector being inserted into to memory, perform 122 /// the insert there, and then read the result back. 123 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx, 124 const SDLoc &dl); 125 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx, 126 const SDLoc &dl); 127 128 /// Return a vector shuffle operation which 129 /// performs the same shuffe in terms of order or result bytes, but on a type 130 /// whose vector element type is narrower than the original shuffle type. 131 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> 132 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl, 133 SDValue N1, SDValue N2, 134 ArrayRef<int> Mask) const; 135 136 bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, 137 bool &NeedInvert, const SDLoc &dl, SDValue &Chain, 138 bool IsSignaling = false); 139 140 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned); 141 142 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32, 143 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, 144 RTLIB::Libcall Call_F128, 145 RTLIB::Libcall Call_PPCF128, 146 SmallVectorImpl<SDValue> &Results); 147 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, 148 RTLIB::Libcall Call_I8, 149 RTLIB::Libcall Call_I16, 150 RTLIB::Libcall Call_I32, 151 RTLIB::Libcall Call_I64, 152 RTLIB::Libcall Call_I128); 153 void ExpandArgFPLibCall(SDNode *Node, 154 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, 155 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, 156 RTLIB::Libcall Call_PPCF128, 157 SmallVectorImpl<SDValue> &Results); 158 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); 159 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); 160 161 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, 162 const SDLoc &dl); 163 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, 164 const SDLoc &dl, SDValue ChainIn); 165 SDValue ExpandBUILD_VECTOR(SDNode *Node); 166 SDValue ExpandSPLAT_VECTOR(SDNode *Node); 167 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node); 168 void ExpandDYNAMIC_STACKALLOC(SDNode *Node, 169 SmallVectorImpl<SDValue> &Results); 170 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL, 171 SDValue Value) const; 172 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL, 173 SDValue NewIntValue) const; 174 SDValue ExpandFCOPYSIGN(SDNode *Node) const; 175 SDValue ExpandFABS(SDNode *Node) const; 176 SDValue ExpandFNEG(SDNode *Node) const; 177 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain); 178 void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl, 179 SmallVectorImpl<SDValue> &Results); 180 void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl, 181 SmallVectorImpl<SDValue> &Results); 182 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl); 183 184 SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl); 185 SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl); 186 SDValue ExpandPARITY(SDValue Op, const SDLoc &dl); 187 188 SDValue ExpandExtractFromVectorThroughStack(SDValue Op); 189 SDValue ExpandInsertToVectorThroughStack(SDValue Op); 190 SDValue ExpandVectorBuildThroughStack(SDNode* Node); 191 192 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP); 193 SDValue ExpandConstant(ConstantSDNode *CP); 194 195 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall 196 bool ExpandNode(SDNode *Node); 197 void ConvertNodeToLibcall(SDNode *Node); 198 void PromoteNode(SDNode *Node); 199 200 public: 201 // Node replacement helpers 202 203 void ReplacedNode(SDNode *N) { 204 LegalizedNodes.erase(N); 205 if (UpdatedNodes) 206 UpdatedNodes->insert(N); 207 } 208 209 void ReplaceNode(SDNode *Old, SDNode *New) { 210 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); 211 dbgs() << " with: "; New->dump(&DAG)); 212 213 assert(Old->getNumValues() == New->getNumValues() && 214 "Replacing one node with another that produces a different number " 215 "of values!"); 216 DAG.ReplaceAllUsesWith(Old, New); 217 if (UpdatedNodes) 218 UpdatedNodes->insert(New); 219 ReplacedNode(Old); 220 } 221 222 void ReplaceNode(SDValue Old, SDValue New) { 223 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); 224 dbgs() << " with: "; New->dump(&DAG)); 225 226 DAG.ReplaceAllUsesWith(Old, New); 227 if (UpdatedNodes) 228 UpdatedNodes->insert(New.getNode()); 229 ReplacedNode(Old.getNode()); 230 } 231 232 void ReplaceNode(SDNode *Old, const SDValue *New) { 233 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG)); 234 235 DAG.ReplaceAllUsesWith(Old, New); 236 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) { 237 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: "); 238 New[i]->dump(&DAG)); 239 if (UpdatedNodes) 240 UpdatedNodes->insert(New[i].getNode()); 241 } 242 ReplacedNode(Old); 243 } 244 245 void ReplaceNodeWithValue(SDValue Old, SDValue New) { 246 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); 247 dbgs() << " with: "; New->dump(&DAG)); 248 249 DAG.ReplaceAllUsesOfValueWith(Old, New); 250 if (UpdatedNodes) 251 UpdatedNodes->insert(New.getNode()); 252 ReplacedNode(Old.getNode()); 253 } 254 }; 255 256 } // end anonymous namespace 257 258 /// Return a vector shuffle operation which 259 /// performs the same shuffle in terms of order or result bytes, but on a type 260 /// whose vector element type is narrower than the original shuffle type. 261 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> 262 SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType( 263 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, 264 ArrayRef<int> Mask) const { 265 unsigned NumMaskElts = VT.getVectorNumElements(); 266 unsigned NumDestElts = NVT.getVectorNumElements(); 267 unsigned NumEltsGrowth = NumDestElts / NumMaskElts; 268 269 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!"); 270 271 if (NumEltsGrowth == 1) 272 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask); 273 274 SmallVector<int, 8> NewMask; 275 for (unsigned i = 0; i != NumMaskElts; ++i) { 276 int Idx = Mask[i]; 277 for (unsigned j = 0; j != NumEltsGrowth; ++j) { 278 if (Idx < 0) 279 NewMask.push_back(-1); 280 else 281 NewMask.push_back(Idx * NumEltsGrowth + j); 282 } 283 } 284 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?"); 285 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?"); 286 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask); 287 } 288 289 /// Expands the ConstantFP node to an integer constant or 290 /// a load from the constant pool. 291 SDValue 292 SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) { 293 bool Extend = false; 294 SDLoc dl(CFP); 295 296 // If a FP immediate is precise when represented as a float and if the 297 // target can do an extending load from float to double, we put it into 298 // the constant pool as a float, even if it's is statically typed as a 299 // double. This shrinks FP constants and canonicalizes them for targets where 300 // an FP extending load is the same cost as a normal load (such as on the x87 301 // fp stack or PPC FP unit). 302 EVT VT = CFP->getValueType(0); 303 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue()); 304 if (!UseCP) { 305 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion"); 306 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl, 307 (VT == MVT::f64) ? MVT::i64 : MVT::i32); 308 } 309 310 APFloat APF = CFP->getValueAPF(); 311 EVT OrigVT = VT; 312 EVT SVT = VT; 313 314 // We don't want to shrink SNaNs. Converting the SNaN back to its real type 315 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ). 316 if (!APF.isSignaling()) { 317 while (SVT != MVT::f32 && SVT != MVT::f16) { 318 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1); 319 if (ConstantFPSDNode::isValueValidForType(SVT, APF) && 320 // Only do this if the target has a native EXTLOAD instruction from 321 // smaller type. 322 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) && 323 TLI.ShouldShrinkFPConstant(OrigVT)) { 324 Type *SType = SVT.getTypeForEVT(*DAG.getContext()); 325 LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType)); 326 VT = SVT; 327 Extend = true; 328 } 329 } 330 } 331 332 SDValue CPIdx = 333 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout())); 334 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign(); 335 if (Extend) { 336 SDValue Result = DAG.getExtLoad( 337 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx, 338 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT, 339 Alignment); 340 return Result; 341 } 342 SDValue Result = DAG.getLoad( 343 OrigVT, dl, DAG.getEntryNode(), CPIdx, 344 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment); 345 return Result; 346 } 347 348 /// Expands the Constant node to a load from the constant pool. 349 SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) { 350 SDLoc dl(CP); 351 EVT VT = CP->getValueType(0); 352 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(), 353 TLI.getPointerTy(DAG.getDataLayout())); 354 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign(); 355 SDValue Result = DAG.getLoad( 356 VT, dl, DAG.getEntryNode(), CPIdx, 357 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment); 358 return Result; 359 } 360 361 /// Some target cannot handle a variable insertion index for the 362 /// INSERT_VECTOR_ELT instruction. In this case, it 363 /// is necessary to spill the vector being inserted into to memory, perform 364 /// the insert there, and then read the result back. 365 SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec, 366 SDValue Val, 367 SDValue Idx, 368 const SDLoc &dl) { 369 SDValue Tmp1 = Vec; 370 SDValue Tmp2 = Val; 371 SDValue Tmp3 = Idx; 372 373 // If the target doesn't support this, we have to spill the input vector 374 // to a temporary stack slot, update the element, then reload it. This is 375 // badness. We could also load the value into a vector register (either 376 // with a "move to register" or "extload into register" instruction, then 377 // permute it into place, if the idx is a constant and if the idx is 378 // supported by the target. 379 EVT VT = Tmp1.getValueType(); 380 EVT EltVT = VT.getVectorElementType(); 381 SDValue StackPtr = DAG.CreateStackTemporary(VT); 382 383 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 384 385 // Store the vector. 386 SDValue Ch = DAG.getStore( 387 DAG.getEntryNode(), dl, Tmp1, StackPtr, 388 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI)); 389 390 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3); 391 392 // Store the scalar value. 393 Ch = DAG.getTruncStore( 394 Ch, dl, Tmp2, StackPtr2, 395 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT); 396 // Load the updated vector. 397 return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack( 398 DAG.getMachineFunction(), SPFI)); 399 } 400 401 SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, 402 SDValue Idx, 403 const SDLoc &dl) { 404 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) { 405 // SCALAR_TO_VECTOR requires that the type of the value being inserted 406 // match the element type of the vector being created, except for 407 // integers in which case the inserted value can be over width. 408 EVT EltVT = Vec.getValueType().getVectorElementType(); 409 if (Val.getValueType() == EltVT || 410 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) { 411 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 412 Vec.getValueType(), Val); 413 414 unsigned NumElts = Vec.getValueType().getVectorNumElements(); 415 // We generate a shuffle of InVec and ScVec, so the shuffle mask 416 // should be 0,1,2,3,4,5... with the appropriate element replaced with 417 // elt 0 of the RHS. 418 SmallVector<int, 8> ShufOps; 419 for (unsigned i = 0; i != NumElts; ++i) 420 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts); 421 422 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps); 423 } 424 } 425 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl); 426 } 427 428 SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) { 429 if (!ISD::isNormalStore(ST)) 430 return SDValue(); 431 432 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n"); 433 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr' 434 // FIXME: move this to the DAG Combiner! Note that we can't regress due 435 // to phase ordering between legalized code and the dag combiner. This 436 // probably means that we need to integrate dag combiner and legalizer 437 // together. 438 // We generally can't do this one for long doubles. 439 SDValue Chain = ST->getChain(); 440 SDValue Ptr = ST->getBasePtr(); 441 SDValue Value = ST->getValue(); 442 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); 443 AAMDNodes AAInfo = ST->getAAInfo(); 444 SDLoc dl(ST); 445 446 // Don't optimise TargetConstantFP 447 if (Value.getOpcode() == ISD::TargetConstantFP) 448 return SDValue(); 449 450 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Value)) { 451 if (CFP->getValueType(0) == MVT::f32 && 452 TLI.isTypeLegal(MVT::i32)) { 453 SDValue Con = DAG.getConstant(CFP->getValueAPF(). 454 bitcastToAPInt().zextOrTrunc(32), 455 SDLoc(CFP), MVT::i32); 456 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), 457 ST->getOriginalAlign(), MMOFlags, AAInfo); 458 } 459 460 if (CFP->getValueType(0) == MVT::f64) { 461 // If this target supports 64-bit registers, do a single 64-bit store. 462 if (TLI.isTypeLegal(MVT::i64)) { 463 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). 464 zextOrTrunc(64), SDLoc(CFP), MVT::i64); 465 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), 466 ST->getOriginalAlign(), MMOFlags, AAInfo); 467 } 468 469 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) { 470 // Otherwise, if the target supports 32-bit registers, use 2 32-bit 471 // stores. If the target supports neither 32- nor 64-bits, this 472 // xform is certainly not worth it. 473 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt(); 474 SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32); 475 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32); 476 if (DAG.getDataLayout().isBigEndian()) 477 std::swap(Lo, Hi); 478 479 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), 480 ST->getOriginalAlign(), MMOFlags, AAInfo); 481 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(4), dl); 482 Hi = DAG.getStore(Chain, dl, Hi, Ptr, 483 ST->getPointerInfo().getWithOffset(4), 484 ST->getOriginalAlign(), MMOFlags, AAInfo); 485 486 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 487 } 488 } 489 } 490 return SDValue(); 491 } 492 493 void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { 494 StoreSDNode *ST = cast<StoreSDNode>(Node); 495 SDValue Chain = ST->getChain(); 496 SDValue Ptr = ST->getBasePtr(); 497 SDLoc dl(Node); 498 499 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); 500 AAMDNodes AAInfo = ST->getAAInfo(); 501 502 if (!ST->isTruncatingStore()) { 503 LLVM_DEBUG(dbgs() << "Legalizing store operation\n"); 504 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) { 505 ReplaceNode(ST, OptStore); 506 return; 507 } 508 509 SDValue Value = ST->getValue(); 510 MVT VT = Value.getSimpleValueType(); 511 switch (TLI.getOperationAction(ISD::STORE, VT)) { 512 default: llvm_unreachable("This action is not supported yet!"); 513 case TargetLowering::Legal: { 514 // If this is an unaligned store and the target doesn't support it, 515 // expand it. 516 EVT MemVT = ST->getMemoryVT(); 517 const DataLayout &DL = DAG.getDataLayout(); 518 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT, 519 *ST->getMemOperand())) { 520 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n"); 521 SDValue Result = TLI.expandUnalignedStore(ST, DAG); 522 ReplaceNode(SDValue(ST, 0), Result); 523 } else 524 LLVM_DEBUG(dbgs() << "Legal store\n"); 525 break; 526 } 527 case TargetLowering::Custom: { 528 LLVM_DEBUG(dbgs() << "Trying custom lowering\n"); 529 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); 530 if (Res && Res != SDValue(Node, 0)) 531 ReplaceNode(SDValue(Node, 0), Res); 532 return; 533 } 534 case TargetLowering::Promote: { 535 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT); 536 assert(NVT.getSizeInBits() == VT.getSizeInBits() && 537 "Can only promote stores to same size type"); 538 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value); 539 SDValue Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), 540 ST->getOriginalAlign(), MMOFlags, AAInfo); 541 ReplaceNode(SDValue(Node, 0), Result); 542 break; 543 } 544 } 545 return; 546 } 547 548 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n"); 549 SDValue Value = ST->getValue(); 550 EVT StVT = ST->getMemoryVT(); 551 TypeSize StWidth = StVT.getSizeInBits(); 552 TypeSize StSize = StVT.getStoreSizeInBits(); 553 auto &DL = DAG.getDataLayout(); 554 555 if (StWidth != StSize) { 556 // Promote to a byte-sized store with upper bits zero if not 557 // storing an integral number of bytes. For example, promote 558 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1) 559 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), StSize.getFixedSize()); 560 Value = DAG.getZeroExtendInReg(Value, dl, StVT); 561 SDValue Result = 562 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT, 563 ST->getOriginalAlign(), MMOFlags, AAInfo); 564 ReplaceNode(SDValue(Node, 0), Result); 565 } else if (!StVT.isVector() && !isPowerOf2_64(StWidth.getFixedSize())) { 566 // If not storing a power-of-2 number of bits, expand as two stores. 567 assert(!StVT.isVector() && "Unsupported truncstore!"); 568 unsigned StWidthBits = StWidth.getFixedSize(); 569 unsigned LogStWidth = Log2_32(StWidthBits); 570 assert(LogStWidth < 32); 571 unsigned RoundWidth = 1 << LogStWidth; 572 assert(RoundWidth < StWidthBits); 573 unsigned ExtraWidth = StWidthBits - RoundWidth; 574 assert(ExtraWidth < RoundWidth); 575 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && 576 "Store size not an integral number of bytes!"); 577 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); 578 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); 579 SDValue Lo, Hi; 580 unsigned IncrementSize; 581 582 if (DL.isLittleEndian()) { 583 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16) 584 // Store the bottom RoundWidth bits. 585 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), 586 RoundVT, ST->getOriginalAlign(), MMOFlags, AAInfo); 587 588 // Store the remaining ExtraWidth bits. 589 IncrementSize = RoundWidth / 8; 590 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); 591 Hi = DAG.getNode( 592 ISD::SRL, dl, Value.getValueType(), Value, 593 DAG.getConstant(RoundWidth, dl, 594 TLI.getShiftAmountTy(Value.getValueType(), DL))); 595 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, 596 ST->getPointerInfo().getWithOffset(IncrementSize), 597 ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo); 598 } else { 599 // Big endian - avoid unaligned stores. 600 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X 601 // Store the top RoundWidth bits. 602 Hi = DAG.getNode( 603 ISD::SRL, dl, Value.getValueType(), Value, 604 DAG.getConstant(ExtraWidth, dl, 605 TLI.getShiftAmountTy(Value.getValueType(), DL))); 606 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), RoundVT, 607 ST->getOriginalAlign(), MMOFlags, AAInfo); 608 609 // Store the remaining ExtraWidth bits. 610 IncrementSize = RoundWidth / 8; 611 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 612 DAG.getConstant(IncrementSize, dl, 613 Ptr.getValueType())); 614 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, 615 ST->getPointerInfo().getWithOffset(IncrementSize), 616 ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo); 617 } 618 619 // The order of the stores doesn't matter. 620 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 621 ReplaceNode(SDValue(Node, 0), Result); 622 } else { 623 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) { 624 default: llvm_unreachable("This action is not supported yet!"); 625 case TargetLowering::Legal: { 626 EVT MemVT = ST->getMemoryVT(); 627 // If this is an unaligned store and the target doesn't support it, 628 // expand it. 629 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT, 630 *ST->getMemOperand())) { 631 SDValue Result = TLI.expandUnalignedStore(ST, DAG); 632 ReplaceNode(SDValue(ST, 0), Result); 633 } 634 break; 635 } 636 case TargetLowering::Custom: { 637 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); 638 if (Res && Res != SDValue(Node, 0)) 639 ReplaceNode(SDValue(Node, 0), Res); 640 return; 641 } 642 case TargetLowering::Expand: 643 assert(!StVT.isVector() && 644 "Vector Stores are handled in LegalizeVectorOps"); 645 646 SDValue Result; 647 648 // TRUNCSTORE:i16 i32 -> STORE i16 649 if (TLI.isTypeLegal(StVT)) { 650 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value); 651 Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), 652 ST->getOriginalAlign(), MMOFlags, AAInfo); 653 } else { 654 // The in-memory type isn't legal. Truncate to the type it would promote 655 // to, and then do a truncstore. 656 Value = DAG.getNode(ISD::TRUNCATE, dl, 657 TLI.getTypeToTransformTo(*DAG.getContext(), StVT), 658 Value); 659 Result = 660 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), StVT, 661 ST->getOriginalAlign(), MMOFlags, AAInfo); 662 } 663 664 ReplaceNode(SDValue(Node, 0), Result); 665 break; 666 } 667 } 668 } 669 670 void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { 671 LoadSDNode *LD = cast<LoadSDNode>(Node); 672 SDValue Chain = LD->getChain(); // The chain. 673 SDValue Ptr = LD->getBasePtr(); // The base pointer. 674 SDValue Value; // The value returned by the load op. 675 SDLoc dl(Node); 676 677 ISD::LoadExtType ExtType = LD->getExtensionType(); 678 if (ExtType == ISD::NON_EXTLOAD) { 679 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n"); 680 MVT VT = Node->getSimpleValueType(0); 681 SDValue RVal = SDValue(Node, 0); 682 SDValue RChain = SDValue(Node, 1); 683 684 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 685 default: llvm_unreachable("This action is not supported yet!"); 686 case TargetLowering::Legal: { 687 EVT MemVT = LD->getMemoryVT(); 688 const DataLayout &DL = DAG.getDataLayout(); 689 // If this is an unaligned load and the target doesn't support it, 690 // expand it. 691 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT, 692 *LD->getMemOperand())) { 693 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG); 694 } 695 break; 696 } 697 case TargetLowering::Custom: 698 if (SDValue Res = TLI.LowerOperation(RVal, DAG)) { 699 RVal = Res; 700 RChain = Res.getValue(1); 701 } 702 break; 703 704 case TargetLowering::Promote: { 705 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); 706 assert(NVT.getSizeInBits() == VT.getSizeInBits() && 707 "Can only promote loads to same size type"); 708 709 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand()); 710 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res); 711 RChain = Res.getValue(1); 712 break; 713 } 714 } 715 if (RChain.getNode() != Node) { 716 assert(RVal.getNode() != Node && "Load must be completely replaced"); 717 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal); 718 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain); 719 if (UpdatedNodes) { 720 UpdatedNodes->insert(RVal.getNode()); 721 UpdatedNodes->insert(RChain.getNode()); 722 } 723 ReplacedNode(Node); 724 } 725 return; 726 } 727 728 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n"); 729 EVT SrcVT = LD->getMemoryVT(); 730 TypeSize SrcWidth = SrcVT.getSizeInBits(); 731 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); 732 AAMDNodes AAInfo = LD->getAAInfo(); 733 734 if (SrcWidth != SrcVT.getStoreSizeInBits() && 735 // Some targets pretend to have an i1 loading operation, and actually 736 // load an i8. This trick is correct for ZEXTLOAD because the top 7 737 // bits are guaranteed to be zero; it helps the optimizers understand 738 // that these bits are zero. It is also useful for EXTLOAD, since it 739 // tells the optimizers that those bits are undefined. It would be 740 // nice to have an effective generic way of getting these benefits... 741 // Until such a way is found, don't insist on promoting i1 here. 742 (SrcVT != MVT::i1 || 743 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) == 744 TargetLowering::Promote)) { 745 // Promote to a byte-sized load if not loading an integral number of 746 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24. 747 unsigned NewWidth = SrcVT.getStoreSizeInBits(); 748 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth); 749 SDValue Ch; 750 751 // The extra bits are guaranteed to be zero, since we stored them that 752 // way. A zext load from NVT thus automatically gives zext from SrcVT. 753 754 ISD::LoadExtType NewExtType = 755 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD; 756 757 SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), 758 Chain, Ptr, LD->getPointerInfo(), NVT, 759 LD->getOriginalAlign(), MMOFlags, AAInfo); 760 761 Ch = Result.getValue(1); // The chain. 762 763 if (ExtType == ISD::SEXTLOAD) 764 // Having the top bits zero doesn't help when sign extending. 765 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, 766 Result.getValueType(), 767 Result, DAG.getValueType(SrcVT)); 768 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType()) 769 // All the top bits are guaranteed to be zero - inform the optimizers. 770 Result = DAG.getNode(ISD::AssertZext, dl, 771 Result.getValueType(), Result, 772 DAG.getValueType(SrcVT)); 773 774 Value = Result; 775 Chain = Ch; 776 } else if (!isPowerOf2_64(SrcWidth.getKnownMinSize())) { 777 // If not loading a power-of-2 number of bits, expand as two loads. 778 assert(!SrcVT.isVector() && "Unsupported extload!"); 779 unsigned SrcWidthBits = SrcWidth.getFixedSize(); 780 unsigned LogSrcWidth = Log2_32(SrcWidthBits); 781 assert(LogSrcWidth < 32); 782 unsigned RoundWidth = 1 << LogSrcWidth; 783 assert(RoundWidth < SrcWidthBits); 784 unsigned ExtraWidth = SrcWidthBits - RoundWidth; 785 assert(ExtraWidth < RoundWidth); 786 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && 787 "Load size not an integral number of bytes!"); 788 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); 789 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); 790 SDValue Lo, Hi, Ch; 791 unsigned IncrementSize; 792 auto &DL = DAG.getDataLayout(); 793 794 if (DL.isLittleEndian()) { 795 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16) 796 // Load the bottom RoundWidth bits. 797 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, 798 LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(), 799 MMOFlags, AAInfo); 800 801 // Load the remaining ExtraWidth bits. 802 IncrementSize = RoundWidth / 8; 803 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); 804 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, 805 LD->getPointerInfo().getWithOffset(IncrementSize), 806 ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo); 807 808 // Build a factor node to remember that this load is independent of 809 // the other one. 810 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 811 Hi.getValue(1)); 812 813 // Move the top bits to the right place. 814 Hi = DAG.getNode( 815 ISD::SHL, dl, Hi.getValueType(), Hi, 816 DAG.getConstant(RoundWidth, dl, 817 TLI.getShiftAmountTy(Hi.getValueType(), DL))); 818 819 // Join the hi and lo parts. 820 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); 821 } else { 822 // Big endian - avoid unaligned loads. 823 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8 824 // Load the top RoundWidth bits. 825 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, 826 LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(), 827 MMOFlags, AAInfo); 828 829 // Load the remaining ExtraWidth bits. 830 IncrementSize = RoundWidth / 8; 831 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); 832 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, 833 LD->getPointerInfo().getWithOffset(IncrementSize), 834 ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo); 835 836 // Build a factor node to remember that this load is independent of 837 // the other one. 838 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 839 Hi.getValue(1)); 840 841 // Move the top bits to the right place. 842 Hi = DAG.getNode( 843 ISD::SHL, dl, Hi.getValueType(), Hi, 844 DAG.getConstant(ExtraWidth, dl, 845 TLI.getShiftAmountTy(Hi.getValueType(), DL))); 846 847 // Join the hi and lo parts. 848 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); 849 } 850 851 Chain = Ch; 852 } else { 853 bool isCustom = false; 854 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0), 855 SrcVT.getSimpleVT())) { 856 default: llvm_unreachable("This action is not supported yet!"); 857 case TargetLowering::Custom: 858 isCustom = true; 859 LLVM_FALLTHROUGH; 860 case TargetLowering::Legal: 861 Value = SDValue(Node, 0); 862 Chain = SDValue(Node, 1); 863 864 if (isCustom) { 865 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { 866 Value = Res; 867 Chain = Res.getValue(1); 868 } 869 } else { 870 // If this is an unaligned load and the target doesn't support it, 871 // expand it. 872 EVT MemVT = LD->getMemoryVT(); 873 const DataLayout &DL = DAG.getDataLayout(); 874 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, 875 *LD->getMemOperand())) { 876 std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG); 877 } 878 } 879 break; 880 881 case TargetLowering::Expand: { 882 EVT DestVT = Node->getValueType(0); 883 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) { 884 // If the source type is not legal, see if there is a legal extload to 885 // an intermediate type that we can then extend further. 886 EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT()); 887 if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT? 888 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) { 889 // If we are loading a legal type, this is a non-extload followed by a 890 // full extend. 891 ISD::LoadExtType MidExtType = 892 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType; 893 894 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr, 895 SrcVT, LD->getMemOperand()); 896 unsigned ExtendOp = 897 ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType); 898 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load); 899 Chain = Load.getValue(1); 900 break; 901 } 902 903 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the 904 // normal undefined upper bits behavior to allow using an in-reg extend 905 // with the illegal FP type, so load as an integer and do the 906 // from-integer conversion. 907 if (SrcVT.getScalarType() == MVT::f16) { 908 EVT ISrcVT = SrcVT.changeTypeToInteger(); 909 EVT IDestVT = DestVT.changeTypeToInteger(); 910 EVT ILoadVT = TLI.getRegisterType(IDestVT.getSimpleVT()); 911 912 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain, 913 Ptr, ISrcVT, LD->getMemOperand()); 914 Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result); 915 Chain = Result.getValue(1); 916 break; 917 } 918 } 919 920 assert(!SrcVT.isVector() && 921 "Vector Loads are handled in LegalizeVectorOps"); 922 923 // FIXME: This does not work for vectors on most targets. Sign- 924 // and zero-extend operations are currently folded into extending 925 // loads, whether they are legal or not, and then we end up here 926 // without any support for legalizing them. 927 assert(ExtType != ISD::EXTLOAD && 928 "EXTLOAD should always be supported!"); 929 // Turn the unsupported load into an EXTLOAD followed by an 930 // explicit zero/sign extend inreg. 931 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl, 932 Node->getValueType(0), 933 Chain, Ptr, SrcVT, 934 LD->getMemOperand()); 935 SDValue ValRes; 936 if (ExtType == ISD::SEXTLOAD) 937 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, 938 Result.getValueType(), 939 Result, DAG.getValueType(SrcVT)); 940 else 941 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT); 942 Value = ValRes; 943 Chain = Result.getValue(1); 944 break; 945 } 946 } 947 } 948 949 // Since loads produce two values, make sure to remember that we legalized 950 // both of them. 951 if (Chain.getNode() != Node) { 952 assert(Value.getNode() != Node && "Load must be completely replaced"); 953 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value); 954 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); 955 if (UpdatedNodes) { 956 UpdatedNodes->insert(Value.getNode()); 957 UpdatedNodes->insert(Chain.getNode()); 958 } 959 ReplacedNode(Node); 960 } 961 } 962 963 /// Return a legal replacement for the given operation, with all legal operands. 964 void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { 965 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG)); 966 967 // Allow illegal target nodes and illegal registers. 968 if (Node->getOpcode() == ISD::TargetConstant || 969 Node->getOpcode() == ISD::Register) 970 return; 971 972 #ifndef NDEBUG 973 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 974 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == 975 TargetLowering::TypeLegal && 976 "Unexpected illegal type!"); 977 978 for (const SDValue &Op : Node->op_values()) 979 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == 980 TargetLowering::TypeLegal || 981 Op.getOpcode() == ISD::TargetConstant || 982 Op.getOpcode() == ISD::Register) && 983 "Unexpected illegal type!"); 984 #endif 985 986 // Figure out the correct action; the way to query this varies by opcode 987 TargetLowering::LegalizeAction Action = TargetLowering::Legal; 988 bool SimpleFinishLegalizing = true; 989 switch (Node->getOpcode()) { 990 case ISD::INTRINSIC_W_CHAIN: 991 case ISD::INTRINSIC_WO_CHAIN: 992 case ISD::INTRINSIC_VOID: 993 case ISD::STACKSAVE: 994 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); 995 break; 996 case ISD::GET_DYNAMIC_AREA_OFFSET: 997 Action = TLI.getOperationAction(Node->getOpcode(), 998 Node->getValueType(0)); 999 break; 1000 case ISD::VAARG: 1001 Action = TLI.getOperationAction(Node->getOpcode(), 1002 Node->getValueType(0)); 1003 if (Action != TargetLowering::Promote) 1004 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); 1005 break; 1006 case ISD::FP_TO_FP16: 1007 case ISD::SINT_TO_FP: 1008 case ISD::UINT_TO_FP: 1009 case ISD::EXTRACT_VECTOR_ELT: 1010 case ISD::LROUND: 1011 case ISD::LLROUND: 1012 case ISD::LRINT: 1013 case ISD::LLRINT: 1014 Action = TLI.getOperationAction(Node->getOpcode(), 1015 Node->getOperand(0).getValueType()); 1016 break; 1017 case ISD::STRICT_FP_TO_FP16: 1018 case ISD::STRICT_SINT_TO_FP: 1019 case ISD::STRICT_UINT_TO_FP: 1020 case ISD::STRICT_LRINT: 1021 case ISD::STRICT_LLRINT: 1022 case ISD::STRICT_LROUND: 1023 case ISD::STRICT_LLROUND: 1024 // These pseudo-ops are the same as the other STRICT_ ops except 1025 // they are registered with setOperationAction() using the input type 1026 // instead of the output type. 1027 Action = TLI.getOperationAction(Node->getOpcode(), 1028 Node->getOperand(1).getValueType()); 1029 break; 1030 case ISD::SIGN_EXTEND_INREG: { 1031 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT(); 1032 Action = TLI.getOperationAction(Node->getOpcode(), InnerType); 1033 break; 1034 } 1035 case ISD::ATOMIC_STORE: 1036 Action = TLI.getOperationAction(Node->getOpcode(), 1037 Node->getOperand(2).getValueType()); 1038 break; 1039 case ISD::SELECT_CC: 1040 case ISD::STRICT_FSETCC: 1041 case ISD::STRICT_FSETCCS: 1042 case ISD::SETCC: 1043 case ISD::BR_CC: { 1044 unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 : 1045 Node->getOpcode() == ISD::STRICT_FSETCC ? 3 : 1046 Node->getOpcode() == ISD::STRICT_FSETCCS ? 3 : 1047 Node->getOpcode() == ISD::SETCC ? 2 : 1; 1048 unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 1049 Node->getOpcode() == ISD::STRICT_FSETCC ? 1 : 1050 Node->getOpcode() == ISD::STRICT_FSETCCS ? 1 : 0; 1051 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType(); 1052 ISD::CondCode CCCode = 1053 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get(); 1054 Action = TLI.getCondCodeAction(CCCode, OpVT); 1055 if (Action == TargetLowering::Legal) { 1056 if (Node->getOpcode() == ISD::SELECT_CC) 1057 Action = TLI.getOperationAction(Node->getOpcode(), 1058 Node->getValueType(0)); 1059 else 1060 Action = TLI.getOperationAction(Node->getOpcode(), OpVT); 1061 } 1062 break; 1063 } 1064 case ISD::LOAD: 1065 case ISD::STORE: 1066 // FIXME: Model these properly. LOAD and STORE are complicated, and 1067 // STORE expects the unlegalized operand in some cases. 1068 SimpleFinishLegalizing = false; 1069 break; 1070 case ISD::CALLSEQ_START: 1071 case ISD::CALLSEQ_END: 1072 // FIXME: This shouldn't be necessary. These nodes have special properties 1073 // dealing with the recursive nature of legalization. Removing this 1074 // special case should be done as part of making LegalizeDAG non-recursive. 1075 SimpleFinishLegalizing = false; 1076 break; 1077 case ISD::EXTRACT_ELEMENT: 1078 case ISD::FLT_ROUNDS_: 1079 case ISD::MERGE_VALUES: 1080 case ISD::EH_RETURN: 1081 case ISD::FRAME_TO_ARGS_OFFSET: 1082 case ISD::EH_DWARF_CFA: 1083 case ISD::EH_SJLJ_SETJMP: 1084 case ISD::EH_SJLJ_LONGJMP: 1085 case ISD::EH_SJLJ_SETUP_DISPATCH: 1086 // These operations lie about being legal: when they claim to be legal, 1087 // they should actually be expanded. 1088 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1089 if (Action == TargetLowering::Legal) 1090 Action = TargetLowering::Expand; 1091 break; 1092 case ISD::INIT_TRAMPOLINE: 1093 case ISD::ADJUST_TRAMPOLINE: 1094 case ISD::FRAMEADDR: 1095 case ISD::RETURNADDR: 1096 case ISD::ADDROFRETURNADDR: 1097 case ISD::SPONENTRY: 1098 // These operations lie about being legal: when they claim to be legal, 1099 // they should actually be custom-lowered. 1100 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1101 if (Action == TargetLowering::Legal) 1102 Action = TargetLowering::Custom; 1103 break; 1104 case ISD::READCYCLECOUNTER: 1105 // READCYCLECOUNTER returns an i64, even if type legalization might have 1106 // expanded that to several smaller types. 1107 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64); 1108 break; 1109 case ISD::READ_REGISTER: 1110 case ISD::WRITE_REGISTER: 1111 // Named register is legal in the DAG, but blocked by register name 1112 // selection if not implemented by target (to chose the correct register) 1113 // They'll be converted to Copy(To/From)Reg. 1114 Action = TargetLowering::Legal; 1115 break; 1116 case ISD::UBSANTRAP: 1117 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1118 if (Action == TargetLowering::Expand) { 1119 // replace ISD::UBSANTRAP with ISD::TRAP 1120 SDValue NewVal; 1121 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), 1122 Node->getOperand(0)); 1123 ReplaceNode(Node, NewVal.getNode()); 1124 LegalizeOp(NewVal.getNode()); 1125 return; 1126 } 1127 break; 1128 case ISD::DEBUGTRAP: 1129 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1130 if (Action == TargetLowering::Expand) { 1131 // replace ISD::DEBUGTRAP with ISD::TRAP 1132 SDValue NewVal; 1133 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), 1134 Node->getOperand(0)); 1135 ReplaceNode(Node, NewVal.getNode()); 1136 LegalizeOp(NewVal.getNode()); 1137 return; 1138 } 1139 break; 1140 case ISD::SADDSAT: 1141 case ISD::UADDSAT: 1142 case ISD::SSUBSAT: 1143 case ISD::USUBSAT: 1144 case ISD::SSHLSAT: 1145 case ISD::USHLSAT: 1146 case ISD::FP_TO_SINT_SAT: 1147 case ISD::FP_TO_UINT_SAT: 1148 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1149 break; 1150 case ISD::SMULFIX: 1151 case ISD::SMULFIXSAT: 1152 case ISD::UMULFIX: 1153 case ISD::UMULFIXSAT: 1154 case ISD::SDIVFIX: 1155 case ISD::SDIVFIXSAT: 1156 case ISD::UDIVFIX: 1157 case ISD::UDIVFIXSAT: { 1158 unsigned Scale = Node->getConstantOperandVal(2); 1159 Action = TLI.getFixedPointOperationAction(Node->getOpcode(), 1160 Node->getValueType(0), Scale); 1161 break; 1162 } 1163 case ISD::MSCATTER: 1164 Action = TLI.getOperationAction(Node->getOpcode(), 1165 cast<MaskedScatterSDNode>(Node)->getValue().getValueType()); 1166 break; 1167 case ISD::MSTORE: 1168 Action = TLI.getOperationAction(Node->getOpcode(), 1169 cast<MaskedStoreSDNode>(Node)->getValue().getValueType()); 1170 break; 1171 case ISD::VECREDUCE_FADD: 1172 case ISD::VECREDUCE_FMUL: 1173 case ISD::VECREDUCE_ADD: 1174 case ISD::VECREDUCE_MUL: 1175 case ISD::VECREDUCE_AND: 1176 case ISD::VECREDUCE_OR: 1177 case ISD::VECREDUCE_XOR: 1178 case ISD::VECREDUCE_SMAX: 1179 case ISD::VECREDUCE_SMIN: 1180 case ISD::VECREDUCE_UMAX: 1181 case ISD::VECREDUCE_UMIN: 1182 case ISD::VECREDUCE_FMAX: 1183 case ISD::VECREDUCE_FMIN: 1184 Action = TLI.getOperationAction( 1185 Node->getOpcode(), Node->getOperand(0).getValueType()); 1186 break; 1187 case ISD::VECREDUCE_SEQ_FADD: 1188 Action = TLI.getOperationAction( 1189 Node->getOpcode(), Node->getOperand(1).getValueType()); 1190 break; 1191 default: 1192 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { 1193 Action = TargetLowering::Legal; 1194 } else { 1195 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1196 } 1197 break; 1198 } 1199 1200 if (SimpleFinishLegalizing) { 1201 SDNode *NewNode = Node; 1202 switch (Node->getOpcode()) { 1203 default: break; 1204 case ISD::SHL: 1205 case ISD::SRL: 1206 case ISD::SRA: 1207 case ISD::ROTL: 1208 case ISD::ROTR: { 1209 // Legalizing shifts/rotates requires adjusting the shift amount 1210 // to the appropriate width. 1211 SDValue Op0 = Node->getOperand(0); 1212 SDValue Op1 = Node->getOperand(1); 1213 if (!Op1.getValueType().isVector()) { 1214 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1); 1215 // The getShiftAmountOperand() may create a new operand node or 1216 // return the existing one. If new operand is created we need 1217 // to update the parent node. 1218 // Do not try to legalize SAO here! It will be automatically legalized 1219 // in the next round. 1220 if (SAO != Op1) 1221 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO); 1222 } 1223 } 1224 break; 1225 case ISD::FSHL: 1226 case ISD::FSHR: 1227 case ISD::SRL_PARTS: 1228 case ISD::SRA_PARTS: 1229 case ISD::SHL_PARTS: { 1230 // Legalizing shifts/rotates requires adjusting the shift amount 1231 // to the appropriate width. 1232 SDValue Op0 = Node->getOperand(0); 1233 SDValue Op1 = Node->getOperand(1); 1234 SDValue Op2 = Node->getOperand(2); 1235 if (!Op2.getValueType().isVector()) { 1236 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2); 1237 // The getShiftAmountOperand() may create a new operand node or 1238 // return the existing one. If new operand is created we need 1239 // to update the parent node. 1240 if (SAO != Op2) 1241 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO); 1242 } 1243 break; 1244 } 1245 } 1246 1247 if (NewNode != Node) { 1248 ReplaceNode(Node, NewNode); 1249 Node = NewNode; 1250 } 1251 switch (Action) { 1252 case TargetLowering::Legal: 1253 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n"); 1254 return; 1255 case TargetLowering::Custom: 1256 LLVM_DEBUG(dbgs() << "Trying custom legalization\n"); 1257 // FIXME: The handling for custom lowering with multiple results is 1258 // a complete mess. 1259 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { 1260 if (!(Res.getNode() != Node || Res.getResNo() != 0)) 1261 return; 1262 1263 if (Node->getNumValues() == 1) { 1264 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n"); 1265 // We can just directly replace this node with the lowered value. 1266 ReplaceNode(SDValue(Node, 0), Res); 1267 return; 1268 } 1269 1270 SmallVector<SDValue, 8> ResultVals; 1271 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 1272 ResultVals.push_back(Res.getValue(i)); 1273 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n"); 1274 ReplaceNode(Node, ResultVals.data()); 1275 return; 1276 } 1277 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n"); 1278 LLVM_FALLTHROUGH; 1279 case TargetLowering::Expand: 1280 if (ExpandNode(Node)) 1281 return; 1282 LLVM_FALLTHROUGH; 1283 case TargetLowering::LibCall: 1284 ConvertNodeToLibcall(Node); 1285 return; 1286 case TargetLowering::Promote: 1287 PromoteNode(Node); 1288 return; 1289 } 1290 } 1291 1292 switch (Node->getOpcode()) { 1293 default: 1294 #ifndef NDEBUG 1295 dbgs() << "NODE: "; 1296 Node->dump( &DAG); 1297 dbgs() << "\n"; 1298 #endif 1299 llvm_unreachable("Do not know how to legalize this operator!"); 1300 1301 case ISD::CALLSEQ_START: 1302 case ISD::CALLSEQ_END: 1303 break; 1304 case ISD::LOAD: 1305 return LegalizeLoadOps(Node); 1306 case ISD::STORE: 1307 return LegalizeStoreOps(Node); 1308 } 1309 } 1310 1311 SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) { 1312 SDValue Vec = Op.getOperand(0); 1313 SDValue Idx = Op.getOperand(1); 1314 SDLoc dl(Op); 1315 1316 // Before we generate a new store to a temporary stack slot, see if there is 1317 // already one that we can use. There often is because when we scalarize 1318 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole 1319 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in 1320 // the vector. If all are expanded here, we don't want one store per vector 1321 // element. 1322 1323 // Caches for hasPredecessorHelper 1324 SmallPtrSet<const SDNode *, 32> Visited; 1325 SmallVector<const SDNode *, 16> Worklist; 1326 Visited.insert(Op.getNode()); 1327 Worklist.push_back(Idx.getNode()); 1328 SDValue StackPtr, Ch; 1329 for (SDNode::use_iterator UI = Vec.getNode()->use_begin(), 1330 UE = Vec.getNode()->use_end(); UI != UE; ++UI) { 1331 SDNode *User = *UI; 1332 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) { 1333 if (ST->isIndexed() || ST->isTruncatingStore() || 1334 ST->getValue() != Vec) 1335 continue; 1336 1337 // Make sure that nothing else could have stored into the destination of 1338 // this store. 1339 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode())) 1340 continue; 1341 1342 // If the index is dependent on the store we will introduce a cycle when 1343 // creating the load (the load uses the index, and by replacing the chain 1344 // we will make the index dependent on the load). Also, the store might be 1345 // dependent on the extractelement and introduce a cycle when creating 1346 // the load. 1347 if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) || 1348 ST->hasPredecessor(Op.getNode())) 1349 continue; 1350 1351 StackPtr = ST->getBasePtr(); 1352 Ch = SDValue(ST, 0); 1353 break; 1354 } 1355 } 1356 1357 EVT VecVT = Vec.getValueType(); 1358 1359 if (!Ch.getNode()) { 1360 // Store the value to a temporary stack slot, then LOAD the returned part. 1361 StackPtr = DAG.CreateStackTemporary(VecVT); 1362 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 1363 MachinePointerInfo()); 1364 } 1365 1366 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); 1367 1368 SDValue NewLoad; 1369 1370 if (Op.getValueType().isVector()) 1371 NewLoad = 1372 DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo()); 1373 else 1374 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, 1375 MachinePointerInfo(), 1376 VecVT.getVectorElementType()); 1377 1378 // Replace the chain going out of the store, by the one out of the load. 1379 DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1)); 1380 1381 // We introduced a cycle though, so update the loads operands, making sure 1382 // to use the original store's chain as an incoming chain. 1383 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(), 1384 NewLoad->op_end()); 1385 NewLoadOperands[0] = Ch; 1386 NewLoad = 1387 SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0); 1388 return NewLoad; 1389 } 1390 1391 SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) { 1392 assert(Op.getValueType().isVector() && "Non-vector insert subvector!"); 1393 1394 SDValue Vec = Op.getOperand(0); 1395 SDValue Part = Op.getOperand(1); 1396 SDValue Idx = Op.getOperand(2); 1397 SDLoc dl(Op); 1398 1399 // Store the value to a temporary stack slot, then LOAD the returned part. 1400 EVT VecVT = Vec.getValueType(); 1401 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 1402 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1403 MachinePointerInfo PtrInfo = 1404 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); 1405 1406 // First store the whole vector. 1407 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo); 1408 1409 // Then store the inserted part. 1410 SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); 1411 1412 // Store the subvector. 1413 Ch = DAG.getStore( 1414 Ch, dl, Part, SubStackPtr, 1415 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction())); 1416 1417 // Finally, load the updated vector. 1418 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo); 1419 } 1420 1421 SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) { 1422 assert((Node->getOpcode() == ISD::BUILD_VECTOR || 1423 Node->getOpcode() == ISD::CONCAT_VECTORS) && 1424 "Unexpected opcode!"); 1425 1426 // We can't handle this case efficiently. Allocate a sufficiently 1427 // aligned object on the stack, store each operand into it, then load 1428 // the result as a vector. 1429 // Create the stack frame object. 1430 EVT VT = Node->getValueType(0); 1431 EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType() 1432 : Node->getOperand(0).getValueType(); 1433 SDLoc dl(Node); 1434 SDValue FIPtr = DAG.CreateStackTemporary(VT); 1435 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); 1436 MachinePointerInfo PtrInfo = 1437 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); 1438 1439 // Emit a store of each element to the stack slot. 1440 SmallVector<SDValue, 8> Stores; 1441 unsigned TypeByteSize = MemVT.getSizeInBits() / 8; 1442 assert(TypeByteSize > 0 && "Vector element type too small for stack store!"); 1443 1444 // If the destination vector element type of a BUILD_VECTOR is narrower than 1445 // the source element type, only store the bits necessary. 1446 bool Truncate = isa<BuildVectorSDNode>(Node) && 1447 MemVT.bitsLT(Node->getOperand(0).getValueType()); 1448 1449 // Store (in the right endianness) the elements to memory. 1450 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 1451 // Ignore undef elements. 1452 if (Node->getOperand(i).isUndef()) continue; 1453 1454 unsigned Offset = TypeByteSize*i; 1455 1456 SDValue Idx = DAG.getMemBasePlusOffset(FIPtr, TypeSize::Fixed(Offset), dl); 1457 1458 if (Truncate) 1459 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl, 1460 Node->getOperand(i), Idx, 1461 PtrInfo.getWithOffset(Offset), MemVT)); 1462 else 1463 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i), 1464 Idx, PtrInfo.getWithOffset(Offset))); 1465 } 1466 1467 SDValue StoreChain; 1468 if (!Stores.empty()) // Not all undef elements? 1469 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); 1470 else 1471 StoreChain = DAG.getEntryNode(); 1472 1473 // Result is a load from the stack slot. 1474 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo); 1475 } 1476 1477 /// Bitcast a floating-point value to an integer value. Only bitcast the part 1478 /// containing the sign bit if the target has no integer value capable of 1479 /// holding all bits of the floating-point value. 1480 void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State, 1481 const SDLoc &DL, 1482 SDValue Value) const { 1483 EVT FloatVT = Value.getValueType(); 1484 unsigned NumBits = FloatVT.getScalarSizeInBits(); 1485 State.FloatVT = FloatVT; 1486 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits); 1487 // Convert to an integer of the same size. 1488 if (TLI.isTypeLegal(IVT)) { 1489 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value); 1490 State.SignMask = APInt::getSignMask(NumBits); 1491 State.SignBit = NumBits - 1; 1492 return; 1493 } 1494 1495 auto &DataLayout = DAG.getDataLayout(); 1496 // Store the float to memory, then load the sign part out as an integer. 1497 MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8); 1498 // First create a temporary that is aligned for both the load and store. 1499 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy); 1500 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1501 // Then store the float to it. 1502 State.FloatPtr = StackPtr; 1503 MachineFunction &MF = DAG.getMachineFunction(); 1504 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI); 1505 State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr, 1506 State.FloatPointerInfo); 1507 1508 SDValue IntPtr; 1509 if (DataLayout.isBigEndian()) { 1510 assert(FloatVT.isByteSized() && "Unsupported floating point type!"); 1511 // Load out a legal integer with the same sign bit as the float. 1512 IntPtr = StackPtr; 1513 State.IntPointerInfo = State.FloatPointerInfo; 1514 } else { 1515 // Advance the pointer so that the loaded byte will contain the sign bit. 1516 unsigned ByteOffset = (NumBits / 8) - 1; 1517 IntPtr = 1518 DAG.getMemBasePlusOffset(StackPtr, TypeSize::Fixed(ByteOffset), DL); 1519 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI, 1520 ByteOffset); 1521 } 1522 1523 State.IntPtr = IntPtr; 1524 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr, 1525 State.IntPointerInfo, MVT::i8); 1526 State.SignMask = APInt::getOneBitSet(LoadTy.getScalarSizeInBits(), 7); 1527 State.SignBit = 7; 1528 } 1529 1530 /// Replace the integer value produced by getSignAsIntValue() with a new value 1531 /// and cast the result back to a floating-point type. 1532 SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State, 1533 const SDLoc &DL, 1534 SDValue NewIntValue) const { 1535 if (!State.Chain) 1536 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue); 1537 1538 // Override the part containing the sign bit in the value stored on the stack. 1539 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr, 1540 State.IntPointerInfo, MVT::i8); 1541 return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr, 1542 State.FloatPointerInfo); 1543 } 1544 1545 SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const { 1546 SDLoc DL(Node); 1547 SDValue Mag = Node->getOperand(0); 1548 SDValue Sign = Node->getOperand(1); 1549 1550 // Get sign bit into an integer value. 1551 FloatSignAsInt SignAsInt; 1552 getSignAsIntValue(SignAsInt, DL, Sign); 1553 1554 EVT IntVT = SignAsInt.IntValue.getValueType(); 1555 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT); 1556 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue, 1557 SignMask); 1558 1559 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X) 1560 EVT FloatVT = Mag.getValueType(); 1561 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) && 1562 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) { 1563 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag); 1564 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue); 1565 SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit, 1566 DAG.getConstant(0, DL, IntVT), ISD::SETNE); 1567 return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue); 1568 } 1569 1570 // Transform Mag value to integer, and clear the sign bit. 1571 FloatSignAsInt MagAsInt; 1572 getSignAsIntValue(MagAsInt, DL, Mag); 1573 EVT MagVT = MagAsInt.IntValue.getValueType(); 1574 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT); 1575 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue, 1576 ClearSignMask); 1577 1578 // Get the signbit at the right position for MagAsInt. 1579 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit; 1580 EVT ShiftVT = IntVT; 1581 if (SignBit.getScalarValueSizeInBits() < 1582 ClearedSign.getScalarValueSizeInBits()) { 1583 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit); 1584 ShiftVT = MagVT; 1585 } 1586 if (ShiftAmount > 0) { 1587 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT); 1588 SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst); 1589 } else if (ShiftAmount < 0) { 1590 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT); 1591 SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst); 1592 } 1593 if (SignBit.getScalarValueSizeInBits() > 1594 ClearedSign.getScalarValueSizeInBits()) { 1595 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit); 1596 } 1597 1598 // Store the part with the modified sign and convert back to float. 1599 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit); 1600 return modifySignAsInt(MagAsInt, DL, CopiedSign); 1601 } 1602 1603 SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const { 1604 // Get the sign bit as an integer. 1605 SDLoc DL(Node); 1606 FloatSignAsInt SignAsInt; 1607 getSignAsIntValue(SignAsInt, DL, Node->getOperand(0)); 1608 EVT IntVT = SignAsInt.IntValue.getValueType(); 1609 1610 // Flip the sign. 1611 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT); 1612 SDValue SignFlip = 1613 DAG.getNode(ISD::XOR, DL, IntVT, SignAsInt.IntValue, SignMask); 1614 1615 // Convert back to float. 1616 return modifySignAsInt(SignAsInt, DL, SignFlip); 1617 } 1618 1619 SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const { 1620 SDLoc DL(Node); 1621 SDValue Value = Node->getOperand(0); 1622 1623 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal. 1624 EVT FloatVT = Value.getValueType(); 1625 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) { 1626 SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT); 1627 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero); 1628 } 1629 1630 // Transform value to integer, clear the sign bit and transform back. 1631 FloatSignAsInt ValueAsInt; 1632 getSignAsIntValue(ValueAsInt, DL, Value); 1633 EVT IntVT = ValueAsInt.IntValue.getValueType(); 1634 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT); 1635 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue, 1636 ClearSignMask); 1637 return modifySignAsInt(ValueAsInt, DL, ClearedSign); 1638 } 1639 1640 void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node, 1641 SmallVectorImpl<SDValue> &Results) { 1642 Register SPReg = TLI.getStackPointerRegisterToSaveRestore(); 1643 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and" 1644 " not tell us which reg is the stack pointer!"); 1645 SDLoc dl(Node); 1646 EVT VT = Node->getValueType(0); 1647 SDValue Tmp1 = SDValue(Node, 0); 1648 SDValue Tmp2 = SDValue(Node, 1); 1649 SDValue Tmp3 = Node->getOperand(2); 1650 SDValue Chain = Tmp1.getOperand(0); 1651 1652 // Chain the dynamic stack allocation so that it doesn't modify the stack 1653 // pointer when other instructions are using the stack. 1654 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl); 1655 1656 SDValue Size = Tmp2.getOperand(1); 1657 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); 1658 Chain = SP.getValue(1); 1659 Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue(); 1660 const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering(); 1661 unsigned Opc = 1662 TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ? 1663 ISD::ADD : ISD::SUB; 1664 1665 Align StackAlign = TFL->getStackAlign(); 1666 Tmp1 = DAG.getNode(Opc, dl, VT, SP, Size); // Value 1667 if (Alignment > StackAlign) 1668 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1, 1669 DAG.getConstant(-Alignment.value(), dl, VT)); 1670 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain 1671 1672 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true), 1673 DAG.getIntPtrConstant(0, dl, true), SDValue(), dl); 1674 1675 Results.push_back(Tmp1); 1676 Results.push_back(Tmp2); 1677 } 1678 1679 /// Legalize a SETCC with given LHS and RHS and condition code CC on the current 1680 /// target. 1681 /// 1682 /// If the SETCC has been legalized using AND / OR, then the legalized node 1683 /// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert 1684 /// will be set to false. 1685 /// 1686 /// If the SETCC has been legalized by using getSetCCSwappedOperands(), 1687 /// then the values of LHS and RHS will be swapped, CC will be set to the 1688 /// new condition, and NeedInvert will be set to false. 1689 /// 1690 /// If the SETCC has been legalized using the inverse condcode, then LHS and 1691 /// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert 1692 /// will be set to true. The caller must invert the result of the SETCC with 1693 /// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect 1694 /// of a true/false result. 1695 /// 1696 /// \returns true if the SetCC has been legalized, false if it hasn't. 1697 bool SelectionDAGLegalize::LegalizeSetCCCondCode( 1698 EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, bool &NeedInvert, 1699 const SDLoc &dl, SDValue &Chain, bool IsSignaling) { 1700 MVT OpVT = LHS.getSimpleValueType(); 1701 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); 1702 NeedInvert = false; 1703 switch (TLI.getCondCodeAction(CCCode, OpVT)) { 1704 default: llvm_unreachable("Unknown condition code action!"); 1705 case TargetLowering::Legal: 1706 // Nothing to do. 1707 break; 1708 case TargetLowering::Expand: { 1709 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode); 1710 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) { 1711 std::swap(LHS, RHS); 1712 CC = DAG.getCondCode(InvCC); 1713 return true; 1714 } 1715 // Swapping operands didn't work. Try inverting the condition. 1716 bool NeedSwap = false; 1717 InvCC = getSetCCInverse(CCCode, OpVT); 1718 if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) { 1719 // If inverting the condition is not enough, try swapping operands 1720 // on top of it. 1721 InvCC = ISD::getSetCCSwappedOperands(InvCC); 1722 NeedSwap = true; 1723 } 1724 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) { 1725 CC = DAG.getCondCode(InvCC); 1726 NeedInvert = true; 1727 if (NeedSwap) 1728 std::swap(LHS, RHS); 1729 return true; 1730 } 1731 1732 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; 1733 unsigned Opc = 0; 1734 switch (CCCode) { 1735 default: llvm_unreachable("Don't know how to expand this condition!"); 1736 case ISD::SETUO: 1737 if (TLI.isCondCodeLegal(ISD::SETUNE, OpVT)) { 1738 CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; 1739 break; 1740 } 1741 assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && 1742 "If SETUE is expanded, SETOEQ or SETUNE must be legal!"); 1743 NeedInvert = true; 1744 LLVM_FALLTHROUGH; 1745 case ISD::SETO: 1746 assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) 1747 && "If SETO is expanded, SETOEQ must be legal!"); 1748 CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break; 1749 case ISD::SETONE: 1750 case ISD::SETUEQ: 1751 // If the SETUO or SETO CC isn't legal, we might be able to use 1752 // SETOGT || SETOLT, inverting the result for SETUEQ. We only need one 1753 // of SETOGT/SETOLT to be legal, the other can be emulated by swapping 1754 // the operands. 1755 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO; 1756 if (!TLI.isCondCodeLegal(CC2, OpVT) && 1757 (TLI.isCondCodeLegal(ISD::SETOGT, OpVT) || 1758 TLI.isCondCodeLegal(ISD::SETOLT, OpVT))) { 1759 CC1 = ISD::SETOGT; 1760 CC2 = ISD::SETOLT; 1761 Opc = ISD::OR; 1762 NeedInvert = ((unsigned)CCCode & 0x8U); 1763 break; 1764 } 1765 LLVM_FALLTHROUGH; 1766 case ISD::SETOEQ: 1767 case ISD::SETOGT: 1768 case ISD::SETOGE: 1769 case ISD::SETOLT: 1770 case ISD::SETOLE: 1771 case ISD::SETUNE: 1772 case ISD::SETUGT: 1773 case ISD::SETUGE: 1774 case ISD::SETULT: 1775 case ISD::SETULE: 1776 // If we are floating point, assign and break, otherwise fall through. 1777 if (!OpVT.isInteger()) { 1778 // We can use the 4th bit to tell if we are the unordered 1779 // or ordered version of the opcode. 1780 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO; 1781 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND; 1782 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10); 1783 break; 1784 } 1785 // Fallthrough if we are unsigned integer. 1786 LLVM_FALLTHROUGH; 1787 case ISD::SETLE: 1788 case ISD::SETGT: 1789 case ISD::SETGE: 1790 case ISD::SETLT: 1791 case ISD::SETNE: 1792 case ISD::SETEQ: 1793 // If all combinations of inverting the condition and swapping operands 1794 // didn't work then we have no means to expand the condition. 1795 llvm_unreachable("Don't know how to expand this condition!"); 1796 } 1797 1798 SDValue SetCC1, SetCC2; 1799 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) { 1800 // If we aren't the ordered or unorder operation, 1801 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS). 1802 SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1, Chain, 1803 IsSignaling); 1804 SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2, Chain, 1805 IsSignaling); 1806 } else { 1807 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS) 1808 SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1, Chain, 1809 IsSignaling); 1810 SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2, Chain, 1811 IsSignaling); 1812 } 1813 if (Chain) 1814 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, SetCC1.getValue(1), 1815 SetCC2.getValue(1)); 1816 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2); 1817 RHS = SDValue(); 1818 CC = SDValue(); 1819 return true; 1820 } 1821 } 1822 return false; 1823 } 1824 1825 /// Emit a store/load combination to the stack. This stores 1826 /// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does 1827 /// a load from the stack slot to DestVT, extending it if needed. 1828 /// The resultant code need not be legal. 1829 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT, 1830 EVT DestVT, const SDLoc &dl) { 1831 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode()); 1832 } 1833 1834 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT, 1835 EVT DestVT, const SDLoc &dl, 1836 SDValue Chain) { 1837 unsigned SrcSize = SrcOp.getValueSizeInBits(); 1838 unsigned SlotSize = SlotVT.getSizeInBits(); 1839 unsigned DestSize = DestVT.getSizeInBits(); 1840 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext()); 1841 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType); 1842 1843 // Don't convert with stack if the load/store is expensive. 1844 if ((SrcSize > SlotSize && 1845 !TLI.isTruncStoreLegalOrCustom(SrcOp.getValueType(), SlotVT)) || 1846 (SlotSize < DestSize && 1847 !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT))) 1848 return SDValue(); 1849 1850 // Create the stack frame object. 1851 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign( 1852 SrcOp.getValueType().getTypeForEVT(*DAG.getContext())); 1853 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT.getStoreSize(), SrcAlign); 1854 1855 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr); 1856 int SPFI = StackPtrFI->getIndex(); 1857 MachinePointerInfo PtrInfo = 1858 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); 1859 1860 // Emit a store to the stack slot. Use a truncstore if the input value is 1861 // later than DestVT. 1862 SDValue Store; 1863 1864 if (SrcSize > SlotSize) 1865 Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo, 1866 SlotVT, SrcAlign); 1867 else { 1868 assert(SrcSize == SlotSize && "Invalid store"); 1869 Store = 1870 DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign); 1871 } 1872 1873 // Result is a load from the stack slot. 1874 if (SlotSize == DestSize) 1875 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign); 1876 1877 assert(SlotSize < DestSize && "Unknown extension!"); 1878 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT, 1879 DestAlign); 1880 } 1881 1882 SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { 1883 SDLoc dl(Node); 1884 // Create a vector sized/aligned stack slot, store the value to element #0, 1885 // then load the whole vector back out. 1886 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); 1887 1888 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr); 1889 int SPFI = StackPtrFI->getIndex(); 1890 1891 SDValue Ch = DAG.getTruncStore( 1892 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr, 1893 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI), 1894 Node->getValueType(0).getVectorElementType()); 1895 return DAG.getLoad( 1896 Node->getValueType(0), dl, Ch, StackPtr, 1897 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI)); 1898 } 1899 1900 static bool 1901 ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, 1902 const TargetLowering &TLI, SDValue &Res) { 1903 unsigned NumElems = Node->getNumOperands(); 1904 SDLoc dl(Node); 1905 EVT VT = Node->getValueType(0); 1906 1907 // Try to group the scalars into pairs, shuffle the pairs together, then 1908 // shuffle the pairs of pairs together, etc. until the vector has 1909 // been built. This will work only if all of the necessary shuffle masks 1910 // are legal. 1911 1912 // We do this in two phases; first to check the legality of the shuffles, 1913 // and next, assuming that all shuffles are legal, to create the new nodes. 1914 for (int Phase = 0; Phase < 2; ++Phase) { 1915 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals, 1916 NewIntermedVals; 1917 for (unsigned i = 0; i < NumElems; ++i) { 1918 SDValue V = Node->getOperand(i); 1919 if (V.isUndef()) 1920 continue; 1921 1922 SDValue Vec; 1923 if (Phase) 1924 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V); 1925 IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i))); 1926 } 1927 1928 while (IntermedVals.size() > 2) { 1929 NewIntermedVals.clear(); 1930 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) { 1931 // This vector and the next vector are shuffled together (simply to 1932 // append the one to the other). 1933 SmallVector<int, 16> ShuffleVec(NumElems, -1); 1934 1935 SmallVector<int, 16> FinalIndices; 1936 FinalIndices.reserve(IntermedVals[i].second.size() + 1937 IntermedVals[i+1].second.size()); 1938 1939 int k = 0; 1940 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f; 1941 ++j, ++k) { 1942 ShuffleVec[k] = j; 1943 FinalIndices.push_back(IntermedVals[i].second[j]); 1944 } 1945 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f; 1946 ++j, ++k) { 1947 ShuffleVec[k] = NumElems + j; 1948 FinalIndices.push_back(IntermedVals[i+1].second[j]); 1949 } 1950 1951 SDValue Shuffle; 1952 if (Phase) 1953 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first, 1954 IntermedVals[i+1].first, 1955 ShuffleVec); 1956 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT)) 1957 return false; 1958 NewIntermedVals.push_back( 1959 std::make_pair(Shuffle, std::move(FinalIndices))); 1960 } 1961 1962 // If we had an odd number of defined values, then append the last 1963 // element to the array of new vectors. 1964 if ((IntermedVals.size() & 1) != 0) 1965 NewIntermedVals.push_back(IntermedVals.back()); 1966 1967 IntermedVals.swap(NewIntermedVals); 1968 } 1969 1970 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 && 1971 "Invalid number of intermediate vectors"); 1972 SDValue Vec1 = IntermedVals[0].first; 1973 SDValue Vec2; 1974 if (IntermedVals.size() > 1) 1975 Vec2 = IntermedVals[1].first; 1976 else if (Phase) 1977 Vec2 = DAG.getUNDEF(VT); 1978 1979 SmallVector<int, 16> ShuffleVec(NumElems, -1); 1980 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i) 1981 ShuffleVec[IntermedVals[0].second[i]] = i; 1982 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i) 1983 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i; 1984 1985 if (Phase) 1986 Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec); 1987 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT)) 1988 return false; 1989 } 1990 1991 return true; 1992 } 1993 1994 /// Expand a BUILD_VECTOR node on targets that don't 1995 /// support the operation, but do support the resultant vector type. 1996 SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { 1997 unsigned NumElems = Node->getNumOperands(); 1998 SDValue Value1, Value2; 1999 SDLoc dl(Node); 2000 EVT VT = Node->getValueType(0); 2001 EVT OpVT = Node->getOperand(0).getValueType(); 2002 EVT EltVT = VT.getVectorElementType(); 2003 2004 // If the only non-undef value is the low element, turn this into a 2005 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X. 2006 bool isOnlyLowElement = true; 2007 bool MoreThanTwoValues = false; 2008 bool isConstant = true; 2009 for (unsigned i = 0; i < NumElems; ++i) { 2010 SDValue V = Node->getOperand(i); 2011 if (V.isUndef()) 2012 continue; 2013 if (i > 0) 2014 isOnlyLowElement = false; 2015 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V)) 2016 isConstant = false; 2017 2018 if (!Value1.getNode()) { 2019 Value1 = V; 2020 } else if (!Value2.getNode()) { 2021 if (V != Value1) 2022 Value2 = V; 2023 } else if (V != Value1 && V != Value2) { 2024 MoreThanTwoValues = true; 2025 } 2026 } 2027 2028 if (!Value1.getNode()) 2029 return DAG.getUNDEF(VT); 2030 2031 if (isOnlyLowElement) 2032 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); 2033 2034 // If all elements are constants, create a load from the constant pool. 2035 if (isConstant) { 2036 SmallVector<Constant*, 16> CV; 2037 for (unsigned i = 0, e = NumElems; i != e; ++i) { 2038 if (ConstantFPSDNode *V = 2039 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) { 2040 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue())); 2041 } else if (ConstantSDNode *V = 2042 dyn_cast<ConstantSDNode>(Node->getOperand(i))) { 2043 if (OpVT==EltVT) 2044 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue())); 2045 else { 2046 // If OpVT and EltVT don't match, EltVT is not legal and the 2047 // element values have been promoted/truncated earlier. Undo this; 2048 // we don't want a v16i8 to become a v16i32 for example. 2049 const ConstantInt *CI = V->getConstantIntValue(); 2050 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()), 2051 CI->getZExtValue())); 2052 } 2053 } else { 2054 assert(Node->getOperand(i).isUndef()); 2055 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext()); 2056 CV.push_back(UndefValue::get(OpNTy)); 2057 } 2058 } 2059 Constant *CP = ConstantVector::get(CV); 2060 SDValue CPIdx = 2061 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout())); 2062 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign(); 2063 return DAG.getLoad( 2064 VT, dl, DAG.getEntryNode(), CPIdx, 2065 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), 2066 Alignment); 2067 } 2068 2069 SmallSet<SDValue, 16> DefinedValues; 2070 for (unsigned i = 0; i < NumElems; ++i) { 2071 if (Node->getOperand(i).isUndef()) 2072 continue; 2073 DefinedValues.insert(Node->getOperand(i)); 2074 } 2075 2076 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) { 2077 if (!MoreThanTwoValues) { 2078 SmallVector<int, 8> ShuffleVec(NumElems, -1); 2079 for (unsigned i = 0; i < NumElems; ++i) { 2080 SDValue V = Node->getOperand(i); 2081 if (V.isUndef()) 2082 continue; 2083 ShuffleVec[i] = V == Value1 ? 0 : NumElems; 2084 } 2085 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) { 2086 // Get the splatted value into the low element of a vector register. 2087 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1); 2088 SDValue Vec2; 2089 if (Value2.getNode()) 2090 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2); 2091 else 2092 Vec2 = DAG.getUNDEF(VT); 2093 2094 // Return shuffle(LowValVec, undef, <0,0,0,0>) 2095 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec); 2096 } 2097 } else { 2098 SDValue Res; 2099 if (ExpandBVWithShuffles(Node, DAG, TLI, Res)) 2100 return Res; 2101 } 2102 } 2103 2104 // Otherwise, we can't handle this case efficiently. 2105 return ExpandVectorBuildThroughStack(Node); 2106 } 2107 2108 SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) { 2109 SDLoc DL(Node); 2110 EVT VT = Node->getValueType(0); 2111 SDValue SplatVal = Node->getOperand(0); 2112 2113 return DAG.getSplatBuildVector(VT, DL, SplatVal); 2114 } 2115 2116 // Expand a node into a call to a libcall. If the result value 2117 // does not fit into a register, return the lo part and set the hi part to the 2118 // by-reg argument. If it does fit into a single register, return the result 2119 // and leave the Hi part unset. 2120 SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, 2121 bool isSigned) { 2122 TargetLowering::ArgListTy Args; 2123 TargetLowering::ArgListEntry Entry; 2124 for (const SDValue &Op : Node->op_values()) { 2125 EVT ArgVT = Op.getValueType(); 2126 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 2127 Entry.Node = Op; 2128 Entry.Ty = ArgTy; 2129 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned); 2130 Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned); 2131 Args.push_back(Entry); 2132 } 2133 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 2134 TLI.getPointerTy(DAG.getDataLayout())); 2135 2136 EVT RetVT = Node->getValueType(0); 2137 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); 2138 2139 // By default, the input chain to this libcall is the entry node of the 2140 // function. If the libcall is going to be emitted as a tail call then 2141 // TLI.isUsedByReturnOnly will change it to the right chain if the return 2142 // node which is being folded has a non-entry input chain. 2143 SDValue InChain = DAG.getEntryNode(); 2144 2145 // isTailCall may be true since the callee does not reference caller stack 2146 // frame. Check if it's in the right position and that the return types match. 2147 SDValue TCChain = InChain; 2148 const Function &F = DAG.getMachineFunction().getFunction(); 2149 bool isTailCall = 2150 TLI.isInTailCallPosition(DAG, Node, TCChain) && 2151 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy()); 2152 if (isTailCall) 2153 InChain = TCChain; 2154 2155 TargetLowering::CallLoweringInfo CLI(DAG); 2156 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned); 2157 CLI.setDebugLoc(SDLoc(Node)) 2158 .setChain(InChain) 2159 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, 2160 std::move(Args)) 2161 .setTailCall(isTailCall) 2162 .setSExtResult(signExtend) 2163 .setZExtResult(!signExtend) 2164 .setIsPostTypeLegalization(true); 2165 2166 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 2167 2168 if (!CallInfo.second.getNode()) { 2169 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG)); 2170 // It's a tailcall, return the chain (which is the DAG root). 2171 return DAG.getRoot(); 2172 } 2173 2174 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG)); 2175 return CallInfo.first; 2176 } 2177 2178 void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, 2179 RTLIB::Libcall Call_F32, 2180 RTLIB::Libcall Call_F64, 2181 RTLIB::Libcall Call_F80, 2182 RTLIB::Libcall Call_F128, 2183 RTLIB::Libcall Call_PPCF128, 2184 SmallVectorImpl<SDValue> &Results) { 2185 RTLIB::Libcall LC; 2186 switch (Node->getSimpleValueType(0).SimpleTy) { 2187 default: llvm_unreachable("Unexpected request for libcall!"); 2188 case MVT::f32: LC = Call_F32; break; 2189 case MVT::f64: LC = Call_F64; break; 2190 case MVT::f80: LC = Call_F80; break; 2191 case MVT::f128: LC = Call_F128; break; 2192 case MVT::ppcf128: LC = Call_PPCF128; break; 2193 } 2194 2195 if (Node->isStrictFPOpcode()) { 2196 EVT RetVT = Node->getValueType(0); 2197 SmallVector<SDValue, 4> Ops(drop_begin(Node->ops())); 2198 TargetLowering::MakeLibCallOptions CallOptions; 2199 // FIXME: This doesn't support tail calls. 2200 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT, 2201 Ops, CallOptions, 2202 SDLoc(Node), 2203 Node->getOperand(0)); 2204 Results.push_back(Tmp.first); 2205 Results.push_back(Tmp.second); 2206 } else { 2207 SDValue Tmp = ExpandLibCall(LC, Node, false); 2208 Results.push_back(Tmp); 2209 } 2210 } 2211 2212 SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned, 2213 RTLIB::Libcall Call_I8, 2214 RTLIB::Libcall Call_I16, 2215 RTLIB::Libcall Call_I32, 2216 RTLIB::Libcall Call_I64, 2217 RTLIB::Libcall Call_I128) { 2218 RTLIB::Libcall LC; 2219 switch (Node->getSimpleValueType(0).SimpleTy) { 2220 default: llvm_unreachable("Unexpected request for libcall!"); 2221 case MVT::i8: LC = Call_I8; break; 2222 case MVT::i16: LC = Call_I16; break; 2223 case MVT::i32: LC = Call_I32; break; 2224 case MVT::i64: LC = Call_I64; break; 2225 case MVT::i128: LC = Call_I128; break; 2226 } 2227 return ExpandLibCall(LC, Node, isSigned); 2228 } 2229 2230 /// Expand the node to a libcall based on first argument type (for instance 2231 /// lround and its variant). 2232 void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node, 2233 RTLIB::Libcall Call_F32, 2234 RTLIB::Libcall Call_F64, 2235 RTLIB::Libcall Call_F80, 2236 RTLIB::Libcall Call_F128, 2237 RTLIB::Libcall Call_PPCF128, 2238 SmallVectorImpl<SDValue> &Results) { 2239 EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType(); 2240 2241 RTLIB::Libcall LC; 2242 switch (InVT.getSimpleVT().SimpleTy) { 2243 default: llvm_unreachable("Unexpected request for libcall!"); 2244 case MVT::f32: LC = Call_F32; break; 2245 case MVT::f64: LC = Call_F64; break; 2246 case MVT::f80: LC = Call_F80; break; 2247 case MVT::f128: LC = Call_F128; break; 2248 case MVT::ppcf128: LC = Call_PPCF128; break; 2249 } 2250 2251 if (Node->isStrictFPOpcode()) { 2252 EVT RetVT = Node->getValueType(0); 2253 SmallVector<SDValue, 4> Ops(Node->op_begin() + 1, Node->op_end()); 2254 TargetLowering::MakeLibCallOptions CallOptions; 2255 // FIXME: This doesn't support tail calls. 2256 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT, 2257 Ops, CallOptions, 2258 SDLoc(Node), 2259 Node->getOperand(0)); 2260 Results.push_back(Tmp.first); 2261 Results.push_back(Tmp.second); 2262 } else { 2263 SDValue Tmp = ExpandLibCall(LC, Node, false); 2264 Results.push_back(Tmp); 2265 } 2266 } 2267 2268 /// Issue libcalls to __{u}divmod to compute div / rem pairs. 2269 void 2270 SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, 2271 SmallVectorImpl<SDValue> &Results) { 2272 unsigned Opcode = Node->getOpcode(); 2273 bool isSigned = Opcode == ISD::SDIVREM; 2274 2275 RTLIB::Libcall LC; 2276 switch (Node->getSimpleValueType(0).SimpleTy) { 2277 default: llvm_unreachable("Unexpected request for libcall!"); 2278 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break; 2279 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break; 2280 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break; 2281 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break; 2282 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break; 2283 } 2284 2285 // The input chain to this libcall is the entry node of the function. 2286 // Legalizing the call will automatically add the previous call to the 2287 // dependence. 2288 SDValue InChain = DAG.getEntryNode(); 2289 2290 EVT RetVT = Node->getValueType(0); 2291 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); 2292 2293 TargetLowering::ArgListTy Args; 2294 TargetLowering::ArgListEntry Entry; 2295 for (const SDValue &Op : Node->op_values()) { 2296 EVT ArgVT = Op.getValueType(); 2297 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 2298 Entry.Node = Op; 2299 Entry.Ty = ArgTy; 2300 Entry.IsSExt = isSigned; 2301 Entry.IsZExt = !isSigned; 2302 Args.push_back(Entry); 2303 } 2304 2305 // Also pass the return address of the remainder. 2306 SDValue FIPtr = DAG.CreateStackTemporary(RetVT); 2307 Entry.Node = FIPtr; 2308 Entry.Ty = RetTy->getPointerTo(); 2309 Entry.IsSExt = isSigned; 2310 Entry.IsZExt = !isSigned; 2311 Args.push_back(Entry); 2312 2313 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 2314 TLI.getPointerTy(DAG.getDataLayout())); 2315 2316 SDLoc dl(Node); 2317 TargetLowering::CallLoweringInfo CLI(DAG); 2318 CLI.setDebugLoc(dl) 2319 .setChain(InChain) 2320 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, 2321 std::move(Args)) 2322 .setSExtResult(isSigned) 2323 .setZExtResult(!isSigned); 2324 2325 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 2326 2327 // Remainder is loaded back from the stack frame. 2328 SDValue Rem = 2329 DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo()); 2330 Results.push_back(CallInfo.first); 2331 Results.push_back(Rem); 2332 } 2333 2334 /// Return true if sincos libcall is available. 2335 static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) { 2336 RTLIB::Libcall LC; 2337 switch (Node->getSimpleValueType(0).SimpleTy) { 2338 default: llvm_unreachable("Unexpected request for libcall!"); 2339 case MVT::f32: LC = RTLIB::SINCOS_F32; break; 2340 case MVT::f64: LC = RTLIB::SINCOS_F64; break; 2341 case MVT::f80: LC = RTLIB::SINCOS_F80; break; 2342 case MVT::f128: LC = RTLIB::SINCOS_F128; break; 2343 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; 2344 } 2345 return TLI.getLibcallName(LC) != nullptr; 2346 } 2347 2348 /// Only issue sincos libcall if both sin and cos are needed. 2349 static bool useSinCos(SDNode *Node) { 2350 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN 2351 ? ISD::FCOS : ISD::FSIN; 2352 2353 SDValue Op0 = Node->getOperand(0); 2354 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(), 2355 UE = Op0.getNode()->use_end(); UI != UE; ++UI) { 2356 SDNode *User = *UI; 2357 if (User == Node) 2358 continue; 2359 // The other user might have been turned into sincos already. 2360 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS) 2361 return true; 2362 } 2363 return false; 2364 } 2365 2366 /// Issue libcalls to sincos to compute sin / cos pairs. 2367 void 2368 SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node, 2369 SmallVectorImpl<SDValue> &Results) { 2370 RTLIB::Libcall LC; 2371 switch (Node->getSimpleValueType(0).SimpleTy) { 2372 default: llvm_unreachable("Unexpected request for libcall!"); 2373 case MVT::f32: LC = RTLIB::SINCOS_F32; break; 2374 case MVT::f64: LC = RTLIB::SINCOS_F64; break; 2375 case MVT::f80: LC = RTLIB::SINCOS_F80; break; 2376 case MVT::f128: LC = RTLIB::SINCOS_F128; break; 2377 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; 2378 } 2379 2380 // The input chain to this libcall is the entry node of the function. 2381 // Legalizing the call will automatically add the previous call to the 2382 // dependence. 2383 SDValue InChain = DAG.getEntryNode(); 2384 2385 EVT RetVT = Node->getValueType(0); 2386 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); 2387 2388 TargetLowering::ArgListTy Args; 2389 TargetLowering::ArgListEntry Entry; 2390 2391 // Pass the argument. 2392 Entry.Node = Node->getOperand(0); 2393 Entry.Ty = RetTy; 2394 Entry.IsSExt = false; 2395 Entry.IsZExt = false; 2396 Args.push_back(Entry); 2397 2398 // Pass the return address of sin. 2399 SDValue SinPtr = DAG.CreateStackTemporary(RetVT); 2400 Entry.Node = SinPtr; 2401 Entry.Ty = RetTy->getPointerTo(); 2402 Entry.IsSExt = false; 2403 Entry.IsZExt = false; 2404 Args.push_back(Entry); 2405 2406 // Also pass the return address of the cos. 2407 SDValue CosPtr = DAG.CreateStackTemporary(RetVT); 2408 Entry.Node = CosPtr; 2409 Entry.Ty = RetTy->getPointerTo(); 2410 Entry.IsSExt = false; 2411 Entry.IsZExt = false; 2412 Args.push_back(Entry); 2413 2414 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 2415 TLI.getPointerTy(DAG.getDataLayout())); 2416 2417 SDLoc dl(Node); 2418 TargetLowering::CallLoweringInfo CLI(DAG); 2419 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee( 2420 TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee, 2421 std::move(Args)); 2422 2423 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 2424 2425 Results.push_back( 2426 DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo())); 2427 Results.push_back( 2428 DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo())); 2429 } 2430 2431 /// This function is responsible for legalizing a 2432 /// INT_TO_FP operation of the specified operand when the target requests that 2433 /// we expand it. At this point, we know that the result and operand types are 2434 /// legal for the target. 2435 SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node, 2436 SDValue &Chain) { 2437 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP || 2438 Node->getOpcode() == ISD::SINT_TO_FP); 2439 EVT DestVT = Node->getValueType(0); 2440 SDLoc dl(Node); 2441 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0; 2442 SDValue Op0 = Node->getOperand(OpNo); 2443 EVT SrcVT = Op0.getValueType(); 2444 2445 // TODO: Should any fast-math-flags be set for the created nodes? 2446 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n"); 2447 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) && 2448 (DestVT.bitsLE(MVT::f64) || 2449 TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND 2450 : ISD::FP_EXTEND, 2451 DestVT))) { 2452 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double " 2453 "expansion\n"); 2454 2455 // Get the stack frame index of a 8 byte buffer. 2456 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64); 2457 2458 SDValue Lo = Op0; 2459 // if signed map to unsigned space 2460 if (isSigned) { 2461 // Invert sign bit (signed to unsigned mapping). 2462 Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo, 2463 DAG.getConstant(0x80000000u, dl, MVT::i32)); 2464 } 2465 // Initial hi portion of constructed double. 2466 SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32); 2467 2468 // If this a big endian target, swap the lo and high data. 2469 if (DAG.getDataLayout().isBigEndian()) 2470 std::swap(Lo, Hi); 2471 2472 SDValue MemChain = DAG.getEntryNode(); 2473 2474 // Store the lo of the constructed double. 2475 SDValue Store1 = DAG.getStore(MemChain, dl, Lo, StackSlot, 2476 MachinePointerInfo()); 2477 // Store the hi of the constructed double. 2478 SDValue HiPtr = DAG.getMemBasePlusOffset(StackSlot, TypeSize::Fixed(4), dl); 2479 SDValue Store2 = 2480 DAG.getStore(MemChain, dl, Hi, HiPtr, MachinePointerInfo()); 2481 MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); 2482 2483 // load the constructed double 2484 SDValue Load = 2485 DAG.getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo()); 2486 // FP constant to bias correct the final result 2487 SDValue Bias = DAG.getConstantFP(isSigned ? 2488 BitsToDouble(0x4330000080000000ULL) : 2489 BitsToDouble(0x4330000000000000ULL), 2490 dl, MVT::f64); 2491 // Subtract the bias and get the final result. 2492 SDValue Sub; 2493 SDValue Result; 2494 if (Node->isStrictFPOpcode()) { 2495 Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other}, 2496 {Node->getOperand(0), Load, Bias}); 2497 Chain = Sub.getValue(1); 2498 if (DestVT != Sub.getValueType()) { 2499 std::pair<SDValue, SDValue> ResultPair; 2500 ResultPair = 2501 DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT); 2502 Result = ResultPair.first; 2503 Chain = ResultPair.second; 2504 } 2505 else 2506 Result = Sub; 2507 } else { 2508 Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias); 2509 Result = DAG.getFPExtendOrRound(Sub, dl, DestVT); 2510 } 2511 return Result; 2512 } 2513 2514 if (isSigned) 2515 return SDValue(); 2516 2517 // TODO: Generalize this for use with other types. 2518 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) || 2519 (SrcVT == MVT::i64 && DestVT == MVT::f64)) { 2520 LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n"); 2521 // For unsigned conversions, convert them to signed conversions using the 2522 // algorithm from the x86_64 __floatundisf in compiler_rt. That method 2523 // should be valid for i32->f32 as well. 2524 2525 // More generally this transform should be valid if there are 3 more bits 2526 // in the integer type than the significand. Rounding uses the first bit 2527 // after the width of the significand and the OR of all bits after that. So 2528 // we need to be able to OR the shifted out bit into one of the bits that 2529 // participate in the OR. 2530 2531 // TODO: This really should be implemented using a branch rather than a 2532 // select. We happen to get lucky and machinesink does the right 2533 // thing most of the time. This would be a good candidate for a 2534 // pseudo-op, or, even better, for whole-function isel. 2535 EVT SetCCVT = getSetCCResultType(SrcVT); 2536 2537 SDValue SignBitTest = DAG.getSetCC( 2538 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT), ISD::SETLT); 2539 2540 EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout()); 2541 SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT); 2542 SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, ShiftConst); 2543 SDValue AndConst = DAG.getConstant(1, dl, SrcVT); 2544 SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Op0, AndConst); 2545 SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr); 2546 2547 SDValue Slow, Fast; 2548 if (Node->isStrictFPOpcode()) { 2549 // In strict mode, we must avoid spurious exceptions, and therefore 2550 // must make sure to only emit a single STRICT_SINT_TO_FP. 2551 SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest, Or, Op0); 2552 Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other }, 2553 { Node->getOperand(0), InCvt }); 2554 Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other }, 2555 { Fast.getValue(1), Fast, Fast }); 2556 Chain = Slow.getValue(1); 2557 // The STRICT_SINT_TO_FP inherits the exception mode from the 2558 // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can 2559 // never raise any exception. 2560 SDNodeFlags Flags; 2561 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept()); 2562 Fast->setFlags(Flags); 2563 Flags.setNoFPExcept(true); 2564 Slow->setFlags(Flags); 2565 } else { 2566 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or); 2567 Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt); 2568 Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); 2569 } 2570 2571 return DAG.getSelect(dl, DestVT, SignBitTest, Slow, Fast); 2572 } 2573 2574 // Don't expand it if there isn't cheap fadd. 2575 if (!TLI.isOperationLegalOrCustom( 2576 Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT)) 2577 return SDValue(); 2578 2579 // The following optimization is valid only if every value in SrcVT (when 2580 // treated as signed) is representable in DestVT. Check that the mantissa 2581 // size of DestVT is >= than the number of bits in SrcVT -1. 2582 assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >= 2583 SrcVT.getSizeInBits() - 1 && 2584 "Cannot perform lossless SINT_TO_FP!"); 2585 2586 SDValue Tmp1; 2587 if (Node->isStrictFPOpcode()) { 2588 Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other }, 2589 { Node->getOperand(0), Op0 }); 2590 } else 2591 Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); 2592 2593 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0, 2594 DAG.getConstant(0, dl, SrcVT), ISD::SETLT); 2595 SDValue Zero = DAG.getIntPtrConstant(0, dl), 2596 Four = DAG.getIntPtrConstant(4, dl); 2597 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(), 2598 SignSet, Four, Zero); 2599 2600 // If the sign bit of the integer is set, the large number will be treated 2601 // as a negative number. To counteract this, the dynamic code adds an 2602 // offset depending on the data type. 2603 uint64_t FF; 2604 switch (SrcVT.getSimpleVT().SimpleTy) { 2605 default: 2606 return SDValue(); 2607 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float) 2608 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float) 2609 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float) 2610 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float) 2611 } 2612 if (DAG.getDataLayout().isLittleEndian()) 2613 FF <<= 32; 2614 Constant *FudgeFactor = ConstantInt::get( 2615 Type::getInt64Ty(*DAG.getContext()), FF); 2616 2617 SDValue CPIdx = 2618 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout())); 2619 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign(); 2620 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset); 2621 Alignment = commonAlignment(Alignment, 4); 2622 SDValue FudgeInReg; 2623 if (DestVT == MVT::f32) 2624 FudgeInReg = DAG.getLoad( 2625 MVT::f32, dl, DAG.getEntryNode(), CPIdx, 2626 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), 2627 Alignment); 2628 else { 2629 SDValue Load = DAG.getExtLoad( 2630 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx, 2631 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32, 2632 Alignment); 2633 HandleSDNode Handle(Load); 2634 LegalizeOp(Load.getNode()); 2635 FudgeInReg = Handle.getValue(); 2636 } 2637 2638 if (Node->isStrictFPOpcode()) { 2639 SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other }, 2640 { Tmp1.getValue(1), Tmp1, FudgeInReg }); 2641 Chain = Result.getValue(1); 2642 return Result; 2643 } 2644 2645 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg); 2646 } 2647 2648 /// This function is responsible for legalizing a 2649 /// *INT_TO_FP operation of the specified operand when the target requests that 2650 /// we promote it. At this point, we know that the result and operand types are 2651 /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP 2652 /// operation that takes a larger input. 2653 void SelectionDAGLegalize::PromoteLegalINT_TO_FP( 2654 SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) { 2655 bool IsStrict = N->isStrictFPOpcode(); 2656 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP || 2657 N->getOpcode() == ISD::STRICT_SINT_TO_FP; 2658 EVT DestVT = N->getValueType(0); 2659 SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0); 2660 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP; 2661 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP; 2662 2663 // First step, figure out the appropriate *INT_TO_FP operation to use. 2664 EVT NewInTy = LegalOp.getValueType(); 2665 2666 unsigned OpToUse = 0; 2667 2668 // Scan for the appropriate larger type to use. 2669 while (true) { 2670 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1); 2671 assert(NewInTy.isInteger() && "Ran out of possibilities!"); 2672 2673 // If the target supports SINT_TO_FP of this type, use it. 2674 if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) { 2675 OpToUse = SIntOp; 2676 break; 2677 } 2678 if (IsSigned) 2679 continue; 2680 2681 // If the target supports UINT_TO_FP of this type, use it. 2682 if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) { 2683 OpToUse = UIntOp; 2684 break; 2685 } 2686 2687 // Otherwise, try a larger type. 2688 } 2689 2690 // Okay, we found the operation and type to use. Zero extend our input to the 2691 // desired type then run the operation on it. 2692 if (IsStrict) { 2693 SDValue Res = 2694 DAG.getNode(OpToUse, dl, {DestVT, MVT::Other}, 2695 {N->getOperand(0), 2696 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, 2697 dl, NewInTy, LegalOp)}); 2698 Results.push_back(Res); 2699 Results.push_back(Res.getValue(1)); 2700 return; 2701 } 2702 2703 Results.push_back( 2704 DAG.getNode(OpToUse, dl, DestVT, 2705 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, 2706 dl, NewInTy, LegalOp))); 2707 } 2708 2709 /// This function is responsible for legalizing a 2710 /// FP_TO_*INT operation of the specified operand when the target requests that 2711 /// we promote it. At this point, we know that the result and operand types are 2712 /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT 2713 /// operation that returns a larger result. 2714 void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl, 2715 SmallVectorImpl<SDValue> &Results) { 2716 bool IsStrict = N->isStrictFPOpcode(); 2717 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT || 2718 N->getOpcode() == ISD::STRICT_FP_TO_SINT; 2719 EVT DestVT = N->getValueType(0); 2720 SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0); 2721 // First step, figure out the appropriate FP_TO*INT operation to use. 2722 EVT NewOutTy = DestVT; 2723 2724 unsigned OpToUse = 0; 2725 2726 // Scan for the appropriate larger type to use. 2727 while (true) { 2728 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1); 2729 assert(NewOutTy.isInteger() && "Ran out of possibilities!"); 2730 2731 // A larger signed type can hold all unsigned values of the requested type, 2732 // so using FP_TO_SINT is valid 2733 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT; 2734 if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy)) 2735 break; 2736 2737 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT. 2738 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT; 2739 if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy)) 2740 break; 2741 2742 // Otherwise, try a larger type. 2743 } 2744 2745 // Okay, we found the operation and type to use. 2746 SDValue Operation; 2747 if (IsStrict) { 2748 SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other); 2749 Operation = DAG.getNode(OpToUse, dl, VTs, N->getOperand(0), LegalOp); 2750 } else 2751 Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp); 2752 2753 // Truncate the result of the extended FP_TO_*INT operation to the desired 2754 // size. 2755 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation); 2756 Results.push_back(Trunc); 2757 if (IsStrict) 2758 Results.push_back(Operation.getValue(1)); 2759 } 2760 2761 /// Promote FP_TO_*INT_SAT operation to a larger result type. At this point 2762 /// the result and operand types are legal and there must be a legal 2763 /// FP_TO_*INT_SAT operation for a larger result type. 2764 SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node, 2765 const SDLoc &dl) { 2766 unsigned Opcode = Node->getOpcode(); 2767 2768 // Scan for the appropriate larger type to use. 2769 EVT NewOutTy = Node->getValueType(0); 2770 while (true) { 2771 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1); 2772 assert(NewOutTy.isInteger() && "Ran out of possibilities!"); 2773 2774 if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy)) 2775 break; 2776 } 2777 2778 // Saturation width is determined by second operand, so we don't have to 2779 // perform any fixup and can directly truncate the result. 2780 SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0), 2781 Node->getOperand(1)); 2782 return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result); 2783 } 2784 2785 /// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts. 2786 SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) { 2787 EVT VT = Op.getValueType(); 2788 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); 2789 unsigned Sz = VT.getScalarSizeInBits(); 2790 2791 SDValue Tmp, Tmp2, Tmp3; 2792 2793 // If we can, perform BSWAP first and then the mask+swap the i4, then i2 2794 // and finally the i1 pairs. 2795 // TODO: We can easily support i4/i2 legal types if any target ever does. 2796 if (Sz >= 8 && isPowerOf2_32(Sz)) { 2797 // Create the masks - repeating the pattern every byte. 2798 APInt MaskHi4 = APInt::getSplat(Sz, APInt(8, 0xF0)); 2799 APInt MaskHi2 = APInt::getSplat(Sz, APInt(8, 0xCC)); 2800 APInt MaskHi1 = APInt::getSplat(Sz, APInt(8, 0xAA)); 2801 APInt MaskLo4 = APInt::getSplat(Sz, APInt(8, 0x0F)); 2802 APInt MaskLo2 = APInt::getSplat(Sz, APInt(8, 0x33)); 2803 APInt MaskLo1 = APInt::getSplat(Sz, APInt(8, 0x55)); 2804 2805 // BSWAP if the type is wider than a single byte. 2806 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op); 2807 2808 // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4) 2809 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT)); 2810 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT)); 2811 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, SHVT)); 2812 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT)); 2813 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); 2814 2815 // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2) 2816 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT)); 2817 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT)); 2818 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, SHVT)); 2819 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT)); 2820 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); 2821 2822 // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1) 2823 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT)); 2824 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT)); 2825 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, SHVT)); 2826 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT)); 2827 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); 2828 return Tmp; 2829 } 2830 2831 Tmp = DAG.getConstant(0, dl, VT); 2832 for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) { 2833 if (I < J) 2834 Tmp2 = 2835 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT)); 2836 else 2837 Tmp2 = 2838 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT)); 2839 2840 APInt Shift(Sz, 1); 2841 Shift <<= J; 2842 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT)); 2843 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2); 2844 } 2845 2846 return Tmp; 2847 } 2848 2849 /// Open code the operations for BSWAP of the specified operation. 2850 SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) { 2851 EVT VT = Op.getValueType(); 2852 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); 2853 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8; 2854 switch (VT.getSimpleVT().getScalarType().SimpleTy) { 2855 default: llvm_unreachable("Unhandled Expand type in BSWAP!"); 2856 case MVT::i16: 2857 // Use a rotate by 8. This can be further expanded if necessary. 2858 return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); 2859 case MVT::i32: 2860 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); 2861 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); 2862 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); 2863 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); 2864 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, 2865 DAG.getConstant(0xFF0000, dl, VT)); 2866 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT)); 2867 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); 2868 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); 2869 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); 2870 case MVT::i64: 2871 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); 2872 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT)); 2873 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); 2874 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); 2875 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); 2876 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); 2877 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT)); 2878 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); 2879 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7, 2880 DAG.getConstant(255ULL<<48, dl, VT)); 2881 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6, 2882 DAG.getConstant(255ULL<<40, dl, VT)); 2883 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5, 2884 DAG.getConstant(255ULL<<32, dl, VT)); 2885 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, 2886 DAG.getConstant(255ULL<<24, dl, VT)); 2887 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, 2888 DAG.getConstant(255ULL<<16, dl, VT)); 2889 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, 2890 DAG.getConstant(255ULL<<8 , dl, VT)); 2891 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); 2892 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); 2893 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); 2894 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); 2895 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); 2896 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); 2897 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); 2898 } 2899 } 2900 2901 /// Open code the operations for PARITY of the specified operation. 2902 SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) { 2903 EVT VT = Op.getValueType(); 2904 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); 2905 unsigned Sz = VT.getScalarSizeInBits(); 2906 2907 // If CTPOP is legal, use it. Otherwise use shifts and xor. 2908 SDValue Result; 2909 if (TLI.isOperationLegal(ISD::CTPOP, VT)) { 2910 Result = DAG.getNode(ISD::CTPOP, dl, VT, Op); 2911 } else { 2912 Result = Op; 2913 for (unsigned i = Log2_32_Ceil(Sz); i != 0;) { 2914 SDValue Shift = DAG.getNode(ISD::SRL, dl, VT, Result, 2915 DAG.getConstant(1ULL << (--i), dl, ShVT)); 2916 Result = DAG.getNode(ISD::XOR, dl, VT, Result, Shift); 2917 } 2918 } 2919 2920 return DAG.getNode(ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT)); 2921 } 2922 2923 bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { 2924 LLVM_DEBUG(dbgs() << "Trying to expand node\n"); 2925 SmallVector<SDValue, 8> Results; 2926 SDLoc dl(Node); 2927 SDValue Tmp1, Tmp2, Tmp3, Tmp4; 2928 bool NeedInvert; 2929 switch (Node->getOpcode()) { 2930 case ISD::ABS: 2931 if (TLI.expandABS(Node, Tmp1, DAG)) 2932 Results.push_back(Tmp1); 2933 break; 2934 case ISD::CTPOP: 2935 if (TLI.expandCTPOP(Node, Tmp1, DAG)) 2936 Results.push_back(Tmp1); 2937 break; 2938 case ISD::CTLZ: 2939 case ISD::CTLZ_ZERO_UNDEF: 2940 if (TLI.expandCTLZ(Node, Tmp1, DAG)) 2941 Results.push_back(Tmp1); 2942 break; 2943 case ISD::CTTZ: 2944 case ISD::CTTZ_ZERO_UNDEF: 2945 if (TLI.expandCTTZ(Node, Tmp1, DAG)) 2946 Results.push_back(Tmp1); 2947 break; 2948 case ISD::BITREVERSE: 2949 Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl)); 2950 break; 2951 case ISD::BSWAP: 2952 Results.push_back(ExpandBSWAP(Node->getOperand(0), dl)); 2953 break; 2954 case ISD::PARITY: 2955 Results.push_back(ExpandPARITY(Node->getOperand(0), dl)); 2956 break; 2957 case ISD::FRAMEADDR: 2958 case ISD::RETURNADDR: 2959 case ISD::FRAME_TO_ARGS_OFFSET: 2960 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); 2961 break; 2962 case ISD::EH_DWARF_CFA: { 2963 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl, 2964 TLI.getPointerTy(DAG.getDataLayout())); 2965 SDValue Offset = DAG.getNode(ISD::ADD, dl, 2966 CfaArg.getValueType(), 2967 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl, 2968 CfaArg.getValueType()), 2969 CfaArg); 2970 SDValue FA = DAG.getNode( 2971 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()), 2972 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()))); 2973 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(), 2974 FA, Offset)); 2975 break; 2976 } 2977 case ISD::FLT_ROUNDS_: 2978 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0))); 2979 Results.push_back(Node->getOperand(0)); 2980 break; 2981 case ISD::EH_RETURN: 2982 case ISD::EH_LABEL: 2983 case ISD::PREFETCH: 2984 case ISD::VAEND: 2985 case ISD::EH_SJLJ_LONGJMP: 2986 // If the target didn't expand these, there's nothing to do, so just 2987 // preserve the chain and be done. 2988 Results.push_back(Node->getOperand(0)); 2989 break; 2990 case ISD::READCYCLECOUNTER: 2991 // If the target didn't expand this, just return 'zero' and preserve the 2992 // chain. 2993 Results.append(Node->getNumValues() - 1, 2994 DAG.getConstant(0, dl, Node->getValueType(0))); 2995 Results.push_back(Node->getOperand(0)); 2996 break; 2997 case ISD::EH_SJLJ_SETJMP: 2998 // If the target didn't expand this, just return 'zero' and preserve the 2999 // chain. 3000 Results.push_back(DAG.getConstant(0, dl, MVT::i32)); 3001 Results.push_back(Node->getOperand(0)); 3002 break; 3003 case ISD::ATOMIC_LOAD: { 3004 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP. 3005 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0)); 3006 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); 3007 SDValue Swap = DAG.getAtomicCmpSwap( 3008 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, 3009 Node->getOperand(0), Node->getOperand(1), Zero, Zero, 3010 cast<AtomicSDNode>(Node)->getMemOperand()); 3011 Results.push_back(Swap.getValue(0)); 3012 Results.push_back(Swap.getValue(1)); 3013 break; 3014 } 3015 case ISD::ATOMIC_STORE: { 3016 // There is no libcall for atomic store; fake it with ATOMIC_SWAP. 3017 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 3018 cast<AtomicSDNode>(Node)->getMemoryVT(), 3019 Node->getOperand(0), 3020 Node->getOperand(1), Node->getOperand(2), 3021 cast<AtomicSDNode>(Node)->getMemOperand()); 3022 Results.push_back(Swap.getValue(1)); 3023 break; 3024 } 3025 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { 3026 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and 3027 // splits out the success value as a comparison. Expanding the resulting 3028 // ATOMIC_CMP_SWAP will produce a libcall. 3029 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); 3030 SDValue Res = DAG.getAtomicCmpSwap( 3031 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, 3032 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2), 3033 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand()); 3034 3035 SDValue ExtRes = Res; 3036 SDValue LHS = Res; 3037 SDValue RHS = Node->getOperand(1); 3038 3039 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT(); 3040 EVT OuterType = Node->getValueType(0); 3041 switch (TLI.getExtendForAtomicOps()) { 3042 case ISD::SIGN_EXTEND: 3043 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res, 3044 DAG.getValueType(AtomicType)); 3045 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType, 3046 Node->getOperand(2), DAG.getValueType(AtomicType)); 3047 ExtRes = LHS; 3048 break; 3049 case ISD::ZERO_EXTEND: 3050 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res, 3051 DAG.getValueType(AtomicType)); 3052 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); 3053 ExtRes = LHS; 3054 break; 3055 case ISD::ANY_EXTEND: 3056 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType); 3057 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); 3058 break; 3059 default: 3060 llvm_unreachable("Invalid atomic op extension"); 3061 } 3062 3063 SDValue Success = 3064 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ); 3065 3066 Results.push_back(ExtRes.getValue(0)); 3067 Results.push_back(Success); 3068 Results.push_back(Res.getValue(1)); 3069 break; 3070 } 3071 case ISD::DYNAMIC_STACKALLOC: 3072 ExpandDYNAMIC_STACKALLOC(Node, Results); 3073 break; 3074 case ISD::MERGE_VALUES: 3075 for (unsigned i = 0; i < Node->getNumValues(); i++) 3076 Results.push_back(Node->getOperand(i)); 3077 break; 3078 case ISD::UNDEF: { 3079 EVT VT = Node->getValueType(0); 3080 if (VT.isInteger()) 3081 Results.push_back(DAG.getConstant(0, dl, VT)); 3082 else { 3083 assert(VT.isFloatingPoint() && "Unknown value type!"); 3084 Results.push_back(DAG.getConstantFP(0, dl, VT)); 3085 } 3086 break; 3087 } 3088 case ISD::STRICT_FP_ROUND: 3089 // When strict mode is enforced we can't do expansion because it 3090 // does not honor the "strict" properties. Only libcall is allowed. 3091 if (TLI.isStrictFPEnabled()) 3092 break; 3093 // We might as well mutate to FP_ROUND when FP_ROUND operation is legal 3094 // since this operation is more efficient than stack operation. 3095 if (TLI.getStrictFPOperationAction(Node->getOpcode(), 3096 Node->getValueType(0)) 3097 == TargetLowering::Legal) 3098 break; 3099 // We fall back to use stack operation when the FP_ROUND operation 3100 // isn't available. 3101 if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0), 3102 Node->getValueType(0), dl, 3103 Node->getOperand(0)))) { 3104 ReplaceNode(Node, Tmp1.getNode()); 3105 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n"); 3106 return true; 3107 } 3108 break; 3109 case ISD::FP_ROUND: 3110 case ISD::BITCAST: 3111 if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), 3112 Node->getValueType(0), dl))) 3113 Results.push_back(Tmp1); 3114 break; 3115 case ISD::STRICT_FP_EXTEND: 3116 // When strict mode is enforced we can't do expansion because it 3117 // does not honor the "strict" properties. Only libcall is allowed. 3118 if (TLI.isStrictFPEnabled()) 3119 break; 3120 // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal 3121 // since this operation is more efficient than stack operation. 3122 if (TLI.getStrictFPOperationAction(Node->getOpcode(), 3123 Node->getValueType(0)) 3124 == TargetLowering::Legal) 3125 break; 3126 // We fall back to use stack operation when the FP_EXTEND operation 3127 // isn't available. 3128 if ((Tmp1 = EmitStackConvert( 3129 Node->getOperand(1), Node->getOperand(1).getValueType(), 3130 Node->getValueType(0), dl, Node->getOperand(0)))) { 3131 ReplaceNode(Node, Tmp1.getNode()); 3132 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n"); 3133 return true; 3134 } 3135 break; 3136 case ISD::FP_EXTEND: 3137 if ((Tmp1 = EmitStackConvert(Node->getOperand(0), 3138 Node->getOperand(0).getValueType(), 3139 Node->getValueType(0), dl))) 3140 Results.push_back(Tmp1); 3141 break; 3142 case ISD::SIGN_EXTEND_INREG: { 3143 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); 3144 EVT VT = Node->getValueType(0); 3145 3146 // An in-register sign-extend of a boolean is a negation: 3147 // 'true' (1) sign-extended is -1. 3148 // 'false' (0) sign-extended is 0. 3149 // However, we must mask the high bits of the source operand because the 3150 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero. 3151 3152 // TODO: Do this for vectors too? 3153 if (ExtraVT.getSizeInBits() == 1) { 3154 SDValue One = DAG.getConstant(1, dl, VT); 3155 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One); 3156 SDValue Zero = DAG.getConstant(0, dl, VT); 3157 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And); 3158 Results.push_back(Neg); 3159 break; 3160 } 3161 3162 // NOTE: we could fall back on load/store here too for targets without 3163 // SRA. However, it is doubtful that any exist. 3164 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); 3165 unsigned BitsDiff = VT.getScalarSizeInBits() - 3166 ExtraVT.getScalarSizeInBits(); 3167 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy); 3168 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), 3169 Node->getOperand(0), ShiftCst); 3170 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); 3171 Results.push_back(Tmp1); 3172 break; 3173 } 3174 case ISD::UINT_TO_FP: 3175 case ISD::STRICT_UINT_TO_FP: 3176 if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) { 3177 Results.push_back(Tmp1); 3178 if (Node->isStrictFPOpcode()) 3179 Results.push_back(Tmp2); 3180 break; 3181 } 3182 LLVM_FALLTHROUGH; 3183 case ISD::SINT_TO_FP: 3184 case ISD::STRICT_SINT_TO_FP: 3185 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) { 3186 Results.push_back(Tmp1); 3187 if (Node->isStrictFPOpcode()) 3188 Results.push_back(Tmp2); 3189 } 3190 break; 3191 case ISD::FP_TO_SINT: 3192 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) 3193 Results.push_back(Tmp1); 3194 break; 3195 case ISD::STRICT_FP_TO_SINT: 3196 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) { 3197 ReplaceNode(Node, Tmp1.getNode()); 3198 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n"); 3199 return true; 3200 } 3201 break; 3202 case ISD::FP_TO_UINT: 3203 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) 3204 Results.push_back(Tmp1); 3205 break; 3206 case ISD::STRICT_FP_TO_UINT: 3207 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) { 3208 // Relink the chain. 3209 DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2); 3210 // Replace the new UINT result. 3211 ReplaceNodeWithValue(SDValue(Node, 0), Tmp1); 3212 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n"); 3213 return true; 3214 } 3215 break; 3216 case ISD::FP_TO_SINT_SAT: 3217 case ISD::FP_TO_UINT_SAT: 3218 Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG)); 3219 break; 3220 case ISD::VAARG: 3221 Results.push_back(DAG.expandVAArg(Node)); 3222 Results.push_back(Results[0].getValue(1)); 3223 break; 3224 case ISD::VACOPY: 3225 Results.push_back(DAG.expandVACopy(Node)); 3226 break; 3227 case ISD::EXTRACT_VECTOR_ELT: 3228 if (Node->getOperand(0).getValueType().getVectorNumElements() == 1) 3229 // This must be an access of the only element. Return it. 3230 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), 3231 Node->getOperand(0)); 3232 else 3233 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0)); 3234 Results.push_back(Tmp1); 3235 break; 3236 case ISD::EXTRACT_SUBVECTOR: 3237 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0))); 3238 break; 3239 case ISD::INSERT_SUBVECTOR: 3240 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0))); 3241 break; 3242 case ISD::CONCAT_VECTORS: 3243 Results.push_back(ExpandVectorBuildThroughStack(Node)); 3244 break; 3245 case ISD::SCALAR_TO_VECTOR: 3246 Results.push_back(ExpandSCALAR_TO_VECTOR(Node)); 3247 break; 3248 case ISD::INSERT_VECTOR_ELT: 3249 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0), 3250 Node->getOperand(1), 3251 Node->getOperand(2), dl)); 3252 break; 3253 case ISD::VECTOR_SHUFFLE: { 3254 SmallVector<int, 32> NewMask; 3255 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); 3256 3257 EVT VT = Node->getValueType(0); 3258 EVT EltVT = VT.getVectorElementType(); 3259 SDValue Op0 = Node->getOperand(0); 3260 SDValue Op1 = Node->getOperand(1); 3261 if (!TLI.isTypeLegal(EltVT)) { 3262 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT); 3263 3264 // BUILD_VECTOR operands are allowed to be wider than the element type. 3265 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept 3266 // it. 3267 if (NewEltVT.bitsLT(EltVT)) { 3268 // Convert shuffle node. 3269 // If original node was v4i64 and the new EltVT is i32, 3270 // cast operands to v8i32 and re-build the mask. 3271 3272 // Calculate new VT, the size of the new VT should be equal to original. 3273 EVT NewVT = 3274 EVT::getVectorVT(*DAG.getContext(), NewEltVT, 3275 VT.getSizeInBits() / NewEltVT.getSizeInBits()); 3276 assert(NewVT.bitsEq(VT)); 3277 3278 // cast operands to new VT 3279 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0); 3280 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1); 3281 3282 // Convert the shuffle mask 3283 unsigned int factor = 3284 NewVT.getVectorNumElements()/VT.getVectorNumElements(); 3285 3286 // EltVT gets smaller 3287 assert(factor > 0); 3288 3289 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { 3290 if (Mask[i] < 0) { 3291 for (unsigned fi = 0; fi < factor; ++fi) 3292 NewMask.push_back(Mask[i]); 3293 } 3294 else { 3295 for (unsigned fi = 0; fi < factor; ++fi) 3296 NewMask.push_back(Mask[i]*factor+fi); 3297 } 3298 } 3299 Mask = NewMask; 3300 VT = NewVT; 3301 } 3302 EltVT = NewEltVT; 3303 } 3304 unsigned NumElems = VT.getVectorNumElements(); 3305 SmallVector<SDValue, 16> Ops; 3306 for (unsigned i = 0; i != NumElems; ++i) { 3307 if (Mask[i] < 0) { 3308 Ops.push_back(DAG.getUNDEF(EltVT)); 3309 continue; 3310 } 3311 unsigned Idx = Mask[i]; 3312 if (Idx < NumElems) 3313 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0, 3314 DAG.getVectorIdxConstant(Idx, dl))); 3315 else 3316 Ops.push_back( 3317 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1, 3318 DAG.getVectorIdxConstant(Idx - NumElems, dl))); 3319 } 3320 3321 Tmp1 = DAG.getBuildVector(VT, dl, Ops); 3322 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type. 3323 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); 3324 Results.push_back(Tmp1); 3325 break; 3326 } 3327 case ISD::EXTRACT_ELEMENT: { 3328 EVT OpTy = Node->getOperand(0).getValueType(); 3329 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) { 3330 // 1 -> Hi 3331 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), 3332 DAG.getConstant(OpTy.getSizeInBits() / 2, dl, 3333 TLI.getShiftAmountTy( 3334 Node->getOperand(0).getValueType(), 3335 DAG.getDataLayout()))); 3336 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); 3337 } else { 3338 // 0 -> Lo 3339 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), 3340 Node->getOperand(0)); 3341 } 3342 Results.push_back(Tmp1); 3343 break; 3344 } 3345 case ISD::STACKSAVE: 3346 // Expand to CopyFromReg if the target set 3347 // StackPointerRegisterToSaveRestore. 3348 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) { 3349 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP, 3350 Node->getValueType(0))); 3351 Results.push_back(Results[0].getValue(1)); 3352 } else { 3353 Results.push_back(DAG.getUNDEF(Node->getValueType(0))); 3354 Results.push_back(Node->getOperand(0)); 3355 } 3356 break; 3357 case ISD::STACKRESTORE: 3358 // Expand to CopyToReg if the target set 3359 // StackPointerRegisterToSaveRestore. 3360 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) { 3361 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP, 3362 Node->getOperand(1))); 3363 } else { 3364 Results.push_back(Node->getOperand(0)); 3365 } 3366 break; 3367 case ISD::GET_DYNAMIC_AREA_OFFSET: 3368 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); 3369 Results.push_back(Results[0].getValue(0)); 3370 break; 3371 case ISD::FCOPYSIGN: 3372 Results.push_back(ExpandFCOPYSIGN(Node)); 3373 break; 3374 case ISD::FNEG: 3375 Results.push_back(ExpandFNEG(Node)); 3376 break; 3377 case ISD::FABS: 3378 Results.push_back(ExpandFABS(Node)); 3379 break; 3380 case ISD::SMIN: 3381 case ISD::SMAX: 3382 case ISD::UMIN: 3383 case ISD::UMAX: { 3384 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B 3385 ISD::CondCode Pred; 3386 switch (Node->getOpcode()) { 3387 default: llvm_unreachable("How did we get here?"); 3388 case ISD::SMAX: Pred = ISD::SETGT; break; 3389 case ISD::SMIN: Pred = ISD::SETLT; break; 3390 case ISD::UMAX: Pred = ISD::SETUGT; break; 3391 case ISD::UMIN: Pred = ISD::SETULT; break; 3392 } 3393 Tmp1 = Node->getOperand(0); 3394 Tmp2 = Node->getOperand(1); 3395 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred); 3396 Results.push_back(Tmp1); 3397 break; 3398 } 3399 case ISD::FMINNUM: 3400 case ISD::FMAXNUM: { 3401 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) 3402 Results.push_back(Expanded); 3403 break; 3404 } 3405 case ISD::FSIN: 3406 case ISD::FCOS: { 3407 EVT VT = Node->getValueType(0); 3408 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin / 3409 // fcos which share the same operand and both are used. 3410 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) || 3411 isSinCosLibcallAvailable(Node, TLI)) 3412 && useSinCos(Node)) { 3413 SDVTList VTs = DAG.getVTList(VT, VT); 3414 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); 3415 if (Node->getOpcode() == ISD::FCOS) 3416 Tmp1 = Tmp1.getValue(1); 3417 Results.push_back(Tmp1); 3418 } 3419 break; 3420 } 3421 case ISD::FMAD: 3422 llvm_unreachable("Illegal fmad should never be formed"); 3423 3424 case ISD::FP16_TO_FP: 3425 if (Node->getValueType(0) != MVT::f32) { 3426 // We can extend to types bigger than f32 in two steps without changing 3427 // the result. Since "f16 -> f32" is much more commonly available, give 3428 // CodeGen the option of emitting that before resorting to a libcall. 3429 SDValue Res = 3430 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0)); 3431 Results.push_back( 3432 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res)); 3433 } 3434 break; 3435 case ISD::STRICT_FP16_TO_FP: 3436 if (Node->getValueType(0) != MVT::f32) { 3437 // We can extend to types bigger than f32 in two steps without changing 3438 // the result. Since "f16 -> f32" is much more commonly available, give 3439 // CodeGen the option of emitting that before resorting to a libcall. 3440 SDValue Res = 3441 DAG.getNode(ISD::STRICT_FP16_TO_FP, dl, {MVT::f32, MVT::Other}, 3442 {Node->getOperand(0), Node->getOperand(1)}); 3443 Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, 3444 {Node->getValueType(0), MVT::Other}, 3445 {Res.getValue(1), Res}); 3446 Results.push_back(Res); 3447 Results.push_back(Res.getValue(1)); 3448 } 3449 break; 3450 case ISD::FP_TO_FP16: 3451 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n"); 3452 if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) { 3453 SDValue Op = Node->getOperand(0); 3454 MVT SVT = Op.getSimpleValueType(); 3455 if ((SVT == MVT::f64 || SVT == MVT::f80) && 3456 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) { 3457 // Under fastmath, we can expand this node into a fround followed by 3458 // a float-half conversion. 3459 SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op, 3460 DAG.getIntPtrConstant(0, dl)); 3461 Results.push_back( 3462 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal)); 3463 } 3464 } 3465 break; 3466 case ISD::ConstantFP: { 3467 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); 3468 // Check to see if this FP immediate is already legal. 3469 // If this is a legal constant, turn it into a TargetConstantFP node. 3470 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0), 3471 DAG.shouldOptForSize())) 3472 Results.push_back(ExpandConstantFP(CFP, true)); 3473 break; 3474 } 3475 case ISD::Constant: { 3476 ConstantSDNode *CP = cast<ConstantSDNode>(Node); 3477 Results.push_back(ExpandConstant(CP)); 3478 break; 3479 } 3480 case ISD::FSUB: { 3481 EVT VT = Node->getValueType(0); 3482 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) && 3483 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) { 3484 const SDNodeFlags Flags = Node->getFlags(); 3485 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); 3486 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags); 3487 Results.push_back(Tmp1); 3488 } 3489 break; 3490 } 3491 case ISD::SUB: { 3492 EVT VT = Node->getValueType(0); 3493 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) && 3494 TLI.isOperationLegalOrCustom(ISD::XOR, VT) && 3495 "Don't know how to expand this subtraction!"); 3496 Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1), 3497 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl, 3498 VT)); 3499 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT)); 3500 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); 3501 break; 3502 } 3503 case ISD::UREM: 3504 case ISD::SREM: 3505 if (TLI.expandREM(Node, Tmp1, DAG)) 3506 Results.push_back(Tmp1); 3507 break; 3508 case ISD::UDIV: 3509 case ISD::SDIV: { 3510 bool isSigned = Node->getOpcode() == ISD::SDIV; 3511 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; 3512 EVT VT = Node->getValueType(0); 3513 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) { 3514 SDVTList VTs = DAG.getVTList(VT, VT); 3515 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0), 3516 Node->getOperand(1)); 3517 Results.push_back(Tmp1); 3518 } 3519 break; 3520 } 3521 case ISD::MULHU: 3522 case ISD::MULHS: { 3523 unsigned ExpandOpcode = 3524 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI; 3525 EVT VT = Node->getValueType(0); 3526 SDVTList VTs = DAG.getVTList(VT, VT); 3527 3528 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0), 3529 Node->getOperand(1)); 3530 Results.push_back(Tmp1.getValue(1)); 3531 break; 3532 } 3533 case ISD::UMUL_LOHI: 3534 case ISD::SMUL_LOHI: { 3535 SDValue LHS = Node->getOperand(0); 3536 SDValue RHS = Node->getOperand(1); 3537 MVT VT = LHS.getSimpleValueType(); 3538 unsigned MULHOpcode = 3539 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS; 3540 3541 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) { 3542 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS)); 3543 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS)); 3544 break; 3545 } 3546 3547 SmallVector<SDValue, 4> Halves; 3548 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext()); 3549 assert(TLI.isTypeLegal(HalfType)); 3550 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves, 3551 HalfType, DAG, 3552 TargetLowering::MulExpansionKind::Always)) { 3553 for (unsigned i = 0; i < 2; ++i) { 3554 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]); 3555 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]); 3556 SDValue Shift = DAG.getConstant( 3557 HalfType.getScalarSizeInBits(), dl, 3558 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout())); 3559 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); 3560 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); 3561 } 3562 break; 3563 } 3564 break; 3565 } 3566 case ISD::MUL: { 3567 EVT VT = Node->getValueType(0); 3568 SDVTList VTs = DAG.getVTList(VT, VT); 3569 // See if multiply or divide can be lowered using two-result operations. 3570 // We just need the low half of the multiply; try both the signed 3571 // and unsigned forms. If the target supports both SMUL_LOHI and 3572 // UMUL_LOHI, form a preference by checking which forms of plain 3573 // MULH it supports. 3574 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT); 3575 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT); 3576 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT); 3577 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT); 3578 unsigned OpToUse = 0; 3579 if (HasSMUL_LOHI && !HasMULHS) { 3580 OpToUse = ISD::SMUL_LOHI; 3581 } else if (HasUMUL_LOHI && !HasMULHU) { 3582 OpToUse = ISD::UMUL_LOHI; 3583 } else if (HasSMUL_LOHI) { 3584 OpToUse = ISD::SMUL_LOHI; 3585 } else if (HasUMUL_LOHI) { 3586 OpToUse = ISD::UMUL_LOHI; 3587 } 3588 if (OpToUse) { 3589 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0), 3590 Node->getOperand(1))); 3591 break; 3592 } 3593 3594 SDValue Lo, Hi; 3595 EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext()); 3596 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) && 3597 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) && 3598 TLI.isOperationLegalOrCustom(ISD::SHL, VT) && 3599 TLI.isOperationLegalOrCustom(ISD::OR, VT) && 3600 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG, 3601 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) { 3602 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); 3603 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi); 3604 SDValue Shift = 3605 DAG.getConstant(HalfType.getSizeInBits(), dl, 3606 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout())); 3607 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); 3608 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); 3609 } 3610 break; 3611 } 3612 case ISD::FSHL: 3613 case ISD::FSHR: 3614 if (TLI.expandFunnelShift(Node, Tmp1, DAG)) 3615 Results.push_back(Tmp1); 3616 break; 3617 case ISD::ROTL: 3618 case ISD::ROTR: 3619 if (TLI.expandROT(Node, true /*AllowVectorOps*/, Tmp1, DAG)) 3620 Results.push_back(Tmp1); 3621 break; 3622 case ISD::SADDSAT: 3623 case ISD::UADDSAT: 3624 case ISD::SSUBSAT: 3625 case ISD::USUBSAT: 3626 Results.push_back(TLI.expandAddSubSat(Node, DAG)); 3627 break; 3628 case ISD::SSHLSAT: 3629 case ISD::USHLSAT: 3630 Results.push_back(TLI.expandShlSat(Node, DAG)); 3631 break; 3632 case ISD::SMULFIX: 3633 case ISD::SMULFIXSAT: 3634 case ISD::UMULFIX: 3635 case ISD::UMULFIXSAT: 3636 Results.push_back(TLI.expandFixedPointMul(Node, DAG)); 3637 break; 3638 case ISD::SDIVFIX: 3639 case ISD::SDIVFIXSAT: 3640 case ISD::UDIVFIX: 3641 case ISD::UDIVFIXSAT: 3642 if (SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(), SDLoc(Node), 3643 Node->getOperand(0), 3644 Node->getOperand(1), 3645 Node->getConstantOperandVal(2), 3646 DAG)) { 3647 Results.push_back(V); 3648 break; 3649 } 3650 // FIXME: We might want to retry here with a wider type if we fail, if that 3651 // type is legal. 3652 // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is 3653 // <= 128 (which is the case for all of the default Embedded-C types), 3654 // we will only get here with types and scales that we could always expand 3655 // if we were allowed to generate libcalls to division functions of illegal 3656 // type. But we cannot do that. 3657 llvm_unreachable("Cannot expand DIVFIX!"); 3658 case ISD::ADDCARRY: 3659 case ISD::SUBCARRY: { 3660 SDValue LHS = Node->getOperand(0); 3661 SDValue RHS = Node->getOperand(1); 3662 SDValue Carry = Node->getOperand(2); 3663 3664 bool IsAdd = Node->getOpcode() == ISD::ADDCARRY; 3665 3666 // Initial add of the 2 operands. 3667 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB; 3668 EVT VT = LHS.getValueType(); 3669 SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS); 3670 3671 // Initial check for overflow. 3672 EVT CarryType = Node->getValueType(1); 3673 EVT SetCCType = getSetCCResultType(Node->getValueType(0)); 3674 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT; 3675 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC); 3676 3677 // Add of the sum and the carry. 3678 SDValue One = DAG.getConstant(1, dl, VT); 3679 SDValue CarryExt = 3680 DAG.getNode(ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One); 3681 SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt); 3682 3683 // Second check for overflow. If we are adding, we can only overflow if the 3684 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0. 3685 // If we are subtracting, we can only overflow if the initial sum is 0 and 3686 // the carry is set, resulting in a new sum of all 1s. 3687 SDValue Zero = DAG.getConstant(0, dl, VT); 3688 SDValue Overflow2 = 3689 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ) 3690 : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ); 3691 Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2, 3692 DAG.getZExtOrTrunc(Carry, dl, SetCCType)); 3693 3694 SDValue ResultCarry = 3695 DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2); 3696 3697 Results.push_back(Sum2); 3698 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT)); 3699 break; 3700 } 3701 case ISD::SADDO: 3702 case ISD::SSUBO: { 3703 SDValue Result, Overflow; 3704 TLI.expandSADDSUBO(Node, Result, Overflow, DAG); 3705 Results.push_back(Result); 3706 Results.push_back(Overflow); 3707 break; 3708 } 3709 case ISD::UADDO: 3710 case ISD::USUBO: { 3711 SDValue Result, Overflow; 3712 TLI.expandUADDSUBO(Node, Result, Overflow, DAG); 3713 Results.push_back(Result); 3714 Results.push_back(Overflow); 3715 break; 3716 } 3717 case ISD::UMULO: 3718 case ISD::SMULO: { 3719 SDValue Result, Overflow; 3720 if (TLI.expandMULO(Node, Result, Overflow, DAG)) { 3721 Results.push_back(Result); 3722 Results.push_back(Overflow); 3723 } 3724 break; 3725 } 3726 case ISD::BUILD_PAIR: { 3727 EVT PairTy = Node->getValueType(0); 3728 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); 3729 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); 3730 Tmp2 = DAG.getNode( 3731 ISD::SHL, dl, PairTy, Tmp2, 3732 DAG.getConstant(PairTy.getSizeInBits() / 2, dl, 3733 TLI.getShiftAmountTy(PairTy, DAG.getDataLayout()))); 3734 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2)); 3735 break; 3736 } 3737 case ISD::SELECT: 3738 Tmp1 = Node->getOperand(0); 3739 Tmp2 = Node->getOperand(1); 3740 Tmp3 = Node->getOperand(2); 3741 if (Tmp1.getOpcode() == ISD::SETCC) { 3742 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1), 3743 Tmp2, Tmp3, 3744 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get()); 3745 } else { 3746 Tmp1 = DAG.getSelectCC(dl, Tmp1, 3747 DAG.getConstant(0, dl, Tmp1.getValueType()), 3748 Tmp2, Tmp3, ISD::SETNE); 3749 } 3750 Tmp1->setFlags(Node->getFlags()); 3751 Results.push_back(Tmp1); 3752 break; 3753 case ISD::BR_JT: { 3754 SDValue Chain = Node->getOperand(0); 3755 SDValue Table = Node->getOperand(1); 3756 SDValue Index = Node->getOperand(2); 3757 3758 const DataLayout &TD = DAG.getDataLayout(); 3759 EVT PTy = TLI.getPointerTy(TD); 3760 3761 unsigned EntrySize = 3762 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD); 3763 3764 // For power-of-two jumptable entry sizes convert multiplication to a shift. 3765 // This transformation needs to be done here since otherwise the MIPS 3766 // backend will end up emitting a three instruction multiply sequence 3767 // instead of a single shift and MSP430 will call a runtime function. 3768 if (llvm::isPowerOf2_32(EntrySize)) 3769 Index = DAG.getNode( 3770 ISD::SHL, dl, Index.getValueType(), Index, 3771 DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType())); 3772 else 3773 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index, 3774 DAG.getConstant(EntrySize, dl, Index.getValueType())); 3775 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(), 3776 Index, Table); 3777 3778 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8); 3779 SDValue LD = DAG.getExtLoad( 3780 ISD::SEXTLOAD, dl, PTy, Chain, Addr, 3781 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT); 3782 Addr = LD; 3783 if (TLI.isJumpTableRelative()) { 3784 // For PIC, the sequence is: 3785 // BRIND(load(Jumptable + index) + RelocBase) 3786 // RelocBase can be JumpTable, GOT or some sort of global base. 3787 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, 3788 TLI.getPICJumpTableRelocBase(Table, DAG)); 3789 } 3790 3791 Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG); 3792 Results.push_back(Tmp1); 3793 break; 3794 } 3795 case ISD::BRCOND: 3796 // Expand brcond's setcc into its constituent parts and create a BR_CC 3797 // Node. 3798 Tmp1 = Node->getOperand(0); 3799 Tmp2 = Node->getOperand(1); 3800 if (Tmp2.getOpcode() == ISD::SETCC) { 3801 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, 3802 Tmp1, Tmp2.getOperand(2), 3803 Tmp2.getOperand(0), Tmp2.getOperand(1), 3804 Node->getOperand(2)); 3805 } else { 3806 // We test only the i1 bit. Skip the AND if UNDEF or another AND. 3807 if (Tmp2.isUndef() || 3808 (Tmp2.getOpcode() == ISD::AND && 3809 isa<ConstantSDNode>(Tmp2.getOperand(1)) && 3810 cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1)) 3811 Tmp3 = Tmp2; 3812 else 3813 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2, 3814 DAG.getConstant(1, dl, Tmp2.getValueType())); 3815 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, 3816 DAG.getCondCode(ISD::SETNE), Tmp3, 3817 DAG.getConstant(0, dl, Tmp3.getValueType()), 3818 Node->getOperand(2)); 3819 } 3820 Results.push_back(Tmp1); 3821 break; 3822 case ISD::SETCC: 3823 case ISD::STRICT_FSETCC: 3824 case ISD::STRICT_FSETCCS: { 3825 bool IsStrict = Node->getOpcode() != ISD::SETCC; 3826 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS; 3827 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); 3828 unsigned Offset = IsStrict ? 1 : 0; 3829 Tmp1 = Node->getOperand(0 + Offset); 3830 Tmp2 = Node->getOperand(1 + Offset); 3831 Tmp3 = Node->getOperand(2 + Offset); 3832 bool Legalized = 3833 LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2, Tmp3, 3834 NeedInvert, dl, Chain, IsSignaling); 3835 3836 if (Legalized) { 3837 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the 3838 // condition code, create a new SETCC node. 3839 if (Tmp3.getNode()) 3840 Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), 3841 Tmp1, Tmp2, Tmp3, Node->getFlags()); 3842 3843 // If we expanded the SETCC by inverting the condition code, then wrap 3844 // the existing SETCC in a NOT to restore the intended condition. 3845 if (NeedInvert) 3846 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0)); 3847 3848 Results.push_back(Tmp1); 3849 if (IsStrict) 3850 Results.push_back(Chain); 3851 3852 break; 3853 } 3854 3855 // FIXME: It seems Legalized is false iff CCCode is Legal. I don't 3856 // understand if this code is useful for strict nodes. 3857 assert(!IsStrict && "Don't know how to expand for strict nodes."); 3858 3859 // Otherwise, SETCC for the given comparison type must be completely 3860 // illegal; expand it into a SELECT_CC. 3861 EVT VT = Node->getValueType(0); 3862 int TrueValue; 3863 switch (TLI.getBooleanContents(Tmp1.getValueType())) { 3864 case TargetLowering::ZeroOrOneBooleanContent: 3865 case TargetLowering::UndefinedBooleanContent: 3866 TrueValue = 1; 3867 break; 3868 case TargetLowering::ZeroOrNegativeOneBooleanContent: 3869 TrueValue = -1; 3870 break; 3871 } 3872 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, 3873 DAG.getConstant(TrueValue, dl, VT), 3874 DAG.getConstant(0, dl, VT), 3875 Tmp3); 3876 Tmp1->setFlags(Node->getFlags()); 3877 Results.push_back(Tmp1); 3878 break; 3879 } 3880 case ISD::SELECT_CC: { 3881 // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS 3882 Tmp1 = Node->getOperand(0); // LHS 3883 Tmp2 = Node->getOperand(1); // RHS 3884 Tmp3 = Node->getOperand(2); // True 3885 Tmp4 = Node->getOperand(3); // False 3886 EVT VT = Node->getValueType(0); 3887 SDValue Chain; 3888 SDValue CC = Node->getOperand(4); 3889 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get(); 3890 3891 if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) { 3892 // If the condition code is legal, then we need to expand this 3893 // node using SETCC and SELECT. 3894 EVT CmpVT = Tmp1.getValueType(); 3895 assert(!TLI.isOperationExpand(ISD::SELECT, VT) && 3896 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be " 3897 "expanded."); 3898 EVT CCVT = getSetCCResultType(CmpVT); 3899 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags()); 3900 Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4)); 3901 break; 3902 } 3903 3904 // SELECT_CC is legal, so the condition code must not be. 3905 bool Legalized = false; 3906 // Try to legalize by inverting the condition. This is for targets that 3907 // might support an ordered version of a condition, but not the unordered 3908 // version (or vice versa). 3909 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, Tmp1.getValueType()); 3910 if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) { 3911 // Use the new condition code and swap true and false 3912 Legalized = true; 3913 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC); 3914 Tmp1->setFlags(Node->getFlags()); 3915 } else { 3916 // If The inverse is not legal, then try to swap the arguments using 3917 // the inverse condition code. 3918 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC); 3919 if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) { 3920 // The swapped inverse condition is legal, so swap true and false, 3921 // lhs and rhs. 3922 Legalized = true; 3923 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC); 3924 Tmp1->setFlags(Node->getFlags()); 3925 } 3926 } 3927 3928 if (!Legalized) { 3929 Legalized = LegalizeSetCCCondCode(getSetCCResultType(Tmp1.getValueType()), 3930 Tmp1, Tmp2, CC, NeedInvert, dl, Chain); 3931 3932 assert(Legalized && "Can't legalize SELECT_CC with legal condition!"); 3933 3934 // If we expanded the SETCC by inverting the condition code, then swap 3935 // the True/False operands to match. 3936 if (NeedInvert) 3937 std::swap(Tmp3, Tmp4); 3938 3939 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the 3940 // condition code, create a new SELECT_CC node. 3941 if (CC.getNode()) { 3942 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), 3943 Tmp1, Tmp2, Tmp3, Tmp4, CC); 3944 } else { 3945 Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType()); 3946 CC = DAG.getCondCode(ISD::SETNE); 3947 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, 3948 Tmp2, Tmp3, Tmp4, CC); 3949 } 3950 Tmp1->setFlags(Node->getFlags()); 3951 } 3952 Results.push_back(Tmp1); 3953 break; 3954 } 3955 case ISD::BR_CC: { 3956 // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS 3957 SDValue Chain; 3958 Tmp1 = Node->getOperand(0); // Chain 3959 Tmp2 = Node->getOperand(2); // LHS 3960 Tmp3 = Node->getOperand(3); // RHS 3961 Tmp4 = Node->getOperand(1); // CC 3962 3963 bool Legalized = 3964 LegalizeSetCCCondCode(getSetCCResultType(Tmp2.getValueType()), Tmp2, 3965 Tmp3, Tmp4, NeedInvert, dl, Chain); 3966 (void)Legalized; 3967 assert(Legalized && "Can't legalize BR_CC with legal condition!"); 3968 3969 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC 3970 // node. 3971 if (Tmp4.getNode()) { 3972 assert(!NeedInvert && "Don't know how to invert BR_CC!"); 3973 3974 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, 3975 Tmp4, Tmp2, Tmp3, Node->getOperand(4)); 3976 } else { 3977 Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType()); 3978 Tmp4 = DAG.getCondCode(NeedInvert ? ISD::SETEQ : ISD::SETNE); 3979 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, 3980 Tmp2, Tmp3, Node->getOperand(4)); 3981 } 3982 Results.push_back(Tmp1); 3983 break; 3984 } 3985 case ISD::BUILD_VECTOR: 3986 Results.push_back(ExpandBUILD_VECTOR(Node)); 3987 break; 3988 case ISD::SPLAT_VECTOR: 3989 Results.push_back(ExpandSPLAT_VECTOR(Node)); 3990 break; 3991 case ISD::SRA: 3992 case ISD::SRL: 3993 case ISD::SHL: { 3994 // Scalarize vector SRA/SRL/SHL. 3995 EVT VT = Node->getValueType(0); 3996 assert(VT.isVector() && "Unable to legalize non-vector shift"); 3997 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal"); 3998 unsigned NumElem = VT.getVectorNumElements(); 3999 4000 SmallVector<SDValue, 8> Scalars; 4001 for (unsigned Idx = 0; Idx < NumElem; Idx++) { 4002 SDValue Ex = 4003 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), 4004 Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl)); 4005 SDValue Sh = 4006 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), 4007 Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl)); 4008 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl, 4009 VT.getScalarType(), Ex, Sh)); 4010 } 4011 4012 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars); 4013 Results.push_back(Result); 4014 break; 4015 } 4016 case ISD::VECREDUCE_FADD: 4017 case ISD::VECREDUCE_FMUL: 4018 case ISD::VECREDUCE_ADD: 4019 case ISD::VECREDUCE_MUL: 4020 case ISD::VECREDUCE_AND: 4021 case ISD::VECREDUCE_OR: 4022 case ISD::VECREDUCE_XOR: 4023 case ISD::VECREDUCE_SMAX: 4024 case ISD::VECREDUCE_SMIN: 4025 case ISD::VECREDUCE_UMAX: 4026 case ISD::VECREDUCE_UMIN: 4027 case ISD::VECREDUCE_FMAX: 4028 case ISD::VECREDUCE_FMIN: 4029 Results.push_back(TLI.expandVecReduce(Node, DAG)); 4030 break; 4031 case ISD::GLOBAL_OFFSET_TABLE: 4032 case ISD::GlobalAddress: 4033 case ISD::GlobalTLSAddress: 4034 case ISD::ExternalSymbol: 4035 case ISD::ConstantPool: 4036 case ISD::JumpTable: 4037 case ISD::INTRINSIC_W_CHAIN: 4038 case ISD::INTRINSIC_WO_CHAIN: 4039 case ISD::INTRINSIC_VOID: 4040 // FIXME: Custom lowering for these operations shouldn't return null! 4041 // Return true so that we don't call ConvertNodeToLibcall which also won't 4042 // do anything. 4043 return true; 4044 } 4045 4046 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) { 4047 // FIXME: We were asked to expand a strict floating-point operation, 4048 // but there is currently no expansion implemented that would preserve 4049 // the "strict" properties. For now, we just fall back to the non-strict 4050 // version if that is legal on the target. The actual mutation of the 4051 // operation will happen in SelectionDAGISel::DoInstructionSelection. 4052 switch (Node->getOpcode()) { 4053 default: 4054 if (TLI.getStrictFPOperationAction(Node->getOpcode(), 4055 Node->getValueType(0)) 4056 == TargetLowering::Legal) 4057 return true; 4058 break; 4059 case ISD::STRICT_FSUB: { 4060 if (TLI.getStrictFPOperationAction( 4061 ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal) 4062 return true; 4063 if (TLI.getStrictFPOperationAction( 4064 ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal) 4065 break; 4066 4067 EVT VT = Node->getValueType(0); 4068 const SDNodeFlags Flags = Node->getFlags(); 4069 SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags); 4070 SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(), 4071 {Node->getOperand(0), Node->getOperand(1), Neg}, 4072 Flags); 4073 4074 Results.push_back(Fadd); 4075 Results.push_back(Fadd.getValue(1)); 4076 break; 4077 } 4078 case ISD::STRICT_SINT_TO_FP: 4079 case ISD::STRICT_UINT_TO_FP: 4080 case ISD::STRICT_LRINT: 4081 case ISD::STRICT_LLRINT: 4082 case ISD::STRICT_LROUND: 4083 case ISD::STRICT_LLROUND: 4084 // These are registered by the operand type instead of the value 4085 // type. Reflect that here. 4086 if (TLI.getStrictFPOperationAction(Node->getOpcode(), 4087 Node->getOperand(1).getValueType()) 4088 == TargetLowering::Legal) 4089 return true; 4090 break; 4091 } 4092 } 4093 4094 // Replace the original node with the legalized result. 4095 if (Results.empty()) { 4096 LLVM_DEBUG(dbgs() << "Cannot expand node\n"); 4097 return false; 4098 } 4099 4100 LLVM_DEBUG(dbgs() << "Successfully expanded node\n"); 4101 ReplaceNode(Node, Results.data()); 4102 return true; 4103 } 4104 4105 void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) { 4106 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n"); 4107 SmallVector<SDValue, 8> Results; 4108 SDLoc dl(Node); 4109 // FIXME: Check flags on the node to see if we can use a finite call. 4110 unsigned Opc = Node->getOpcode(); 4111 switch (Opc) { 4112 case ISD::ATOMIC_FENCE: { 4113 // If the target didn't lower this, lower it to '__sync_synchronize()' call 4114 // FIXME: handle "fence singlethread" more efficiently. 4115 TargetLowering::ArgListTy Args; 4116 4117 TargetLowering::CallLoweringInfo CLI(DAG); 4118 CLI.setDebugLoc(dl) 4119 .setChain(Node->getOperand(0)) 4120 .setLibCallee( 4121 CallingConv::C, Type::getVoidTy(*DAG.getContext()), 4122 DAG.getExternalSymbol("__sync_synchronize", 4123 TLI.getPointerTy(DAG.getDataLayout())), 4124 std::move(Args)); 4125 4126 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); 4127 4128 Results.push_back(CallResult.second); 4129 break; 4130 } 4131 // By default, atomic intrinsics are marked Legal and lowered. Targets 4132 // which don't support them directly, however, may want libcalls, in which 4133 // case they mark them Expand, and we get here. 4134 case ISD::ATOMIC_SWAP: 4135 case ISD::ATOMIC_LOAD_ADD: 4136 case ISD::ATOMIC_LOAD_SUB: 4137 case ISD::ATOMIC_LOAD_AND: 4138 case ISD::ATOMIC_LOAD_CLR: 4139 case ISD::ATOMIC_LOAD_OR: 4140 case ISD::ATOMIC_LOAD_XOR: 4141 case ISD::ATOMIC_LOAD_NAND: 4142 case ISD::ATOMIC_LOAD_MIN: 4143 case ISD::ATOMIC_LOAD_MAX: 4144 case ISD::ATOMIC_LOAD_UMIN: 4145 case ISD::ATOMIC_LOAD_UMAX: 4146 case ISD::ATOMIC_CMP_SWAP: { 4147 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); 4148 AtomicOrdering Order = cast<AtomicSDNode>(Node)->getOrdering(); 4149 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT); 4150 EVT RetVT = Node->getValueType(0); 4151 TargetLowering::MakeLibCallOptions CallOptions; 4152 SmallVector<SDValue, 4> Ops; 4153 if (TLI.getLibcallName(LC)) { 4154 // If outline atomic available, prepare its arguments and expand. 4155 Ops.append(Node->op_begin() + 2, Node->op_end()); 4156 Ops.push_back(Node->getOperand(1)); 4157 4158 } else { 4159 LC = RTLIB::getSYNC(Opc, VT); 4160 assert(LC != RTLIB::UNKNOWN_LIBCALL && 4161 "Unexpected atomic op or value type!"); 4162 // Arguments for expansion to sync libcall 4163 Ops.append(Node->op_begin() + 1, Node->op_end()); 4164 } 4165 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT, 4166 Ops, CallOptions, 4167 SDLoc(Node), 4168 Node->getOperand(0)); 4169 Results.push_back(Tmp.first); 4170 Results.push_back(Tmp.second); 4171 break; 4172 } 4173 case ISD::TRAP: { 4174 // If this operation is not supported, lower it to 'abort()' call 4175 TargetLowering::ArgListTy Args; 4176 TargetLowering::CallLoweringInfo CLI(DAG); 4177 CLI.setDebugLoc(dl) 4178 .setChain(Node->getOperand(0)) 4179 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()), 4180 DAG.getExternalSymbol( 4181 "abort", TLI.getPointerTy(DAG.getDataLayout())), 4182 std::move(Args)); 4183 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); 4184 4185 Results.push_back(CallResult.second); 4186 break; 4187 } 4188 case ISD::FMINNUM: 4189 case ISD::STRICT_FMINNUM: 4190 ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64, 4191 RTLIB::FMIN_F80, RTLIB::FMIN_F128, 4192 RTLIB::FMIN_PPCF128, Results); 4193 break; 4194 case ISD::FMAXNUM: 4195 case ISD::STRICT_FMAXNUM: 4196 ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64, 4197 RTLIB::FMAX_F80, RTLIB::FMAX_F128, 4198 RTLIB::FMAX_PPCF128, Results); 4199 break; 4200 case ISD::FSQRT: 4201 case ISD::STRICT_FSQRT: 4202 ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64, 4203 RTLIB::SQRT_F80, RTLIB::SQRT_F128, 4204 RTLIB::SQRT_PPCF128, Results); 4205 break; 4206 case ISD::FCBRT: 4207 ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64, 4208 RTLIB::CBRT_F80, RTLIB::CBRT_F128, 4209 RTLIB::CBRT_PPCF128, Results); 4210 break; 4211 case ISD::FSIN: 4212 case ISD::STRICT_FSIN: 4213 ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64, 4214 RTLIB::SIN_F80, RTLIB::SIN_F128, 4215 RTLIB::SIN_PPCF128, Results); 4216 break; 4217 case ISD::FCOS: 4218 case ISD::STRICT_FCOS: 4219 ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64, 4220 RTLIB::COS_F80, RTLIB::COS_F128, 4221 RTLIB::COS_PPCF128, Results); 4222 break; 4223 case ISD::FSINCOS: 4224 // Expand into sincos libcall. 4225 ExpandSinCosLibCall(Node, Results); 4226 break; 4227 case ISD::FLOG: 4228 case ISD::STRICT_FLOG: 4229 ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80, 4230 RTLIB::LOG_F128, RTLIB::LOG_PPCF128, Results); 4231 break; 4232 case ISD::FLOG2: 4233 case ISD::STRICT_FLOG2: 4234 ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80, 4235 RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128, Results); 4236 break; 4237 case ISD::FLOG10: 4238 case ISD::STRICT_FLOG10: 4239 ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80, 4240 RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128, Results); 4241 break; 4242 case ISD::FEXP: 4243 case ISD::STRICT_FEXP: 4244 ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80, 4245 RTLIB::EXP_F128, RTLIB::EXP_PPCF128, Results); 4246 break; 4247 case ISD::FEXP2: 4248 case ISD::STRICT_FEXP2: 4249 ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80, 4250 RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128, Results); 4251 break; 4252 case ISD::FTRUNC: 4253 case ISD::STRICT_FTRUNC: 4254 ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, 4255 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128, 4256 RTLIB::TRUNC_PPCF128, Results); 4257 break; 4258 case ISD::FFLOOR: 4259 case ISD::STRICT_FFLOOR: 4260 ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, 4261 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128, 4262 RTLIB::FLOOR_PPCF128, Results); 4263 break; 4264 case ISD::FCEIL: 4265 case ISD::STRICT_FCEIL: 4266 ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64, 4267 RTLIB::CEIL_F80, RTLIB::CEIL_F128, 4268 RTLIB::CEIL_PPCF128, Results); 4269 break; 4270 case ISD::FRINT: 4271 case ISD::STRICT_FRINT: 4272 ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64, 4273 RTLIB::RINT_F80, RTLIB::RINT_F128, 4274 RTLIB::RINT_PPCF128, Results); 4275 break; 4276 case ISD::FNEARBYINT: 4277 case ISD::STRICT_FNEARBYINT: 4278 ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32, 4279 RTLIB::NEARBYINT_F64, 4280 RTLIB::NEARBYINT_F80, 4281 RTLIB::NEARBYINT_F128, 4282 RTLIB::NEARBYINT_PPCF128, Results); 4283 break; 4284 case ISD::FROUND: 4285 case ISD::STRICT_FROUND: 4286 ExpandFPLibCall(Node, RTLIB::ROUND_F32, 4287 RTLIB::ROUND_F64, 4288 RTLIB::ROUND_F80, 4289 RTLIB::ROUND_F128, 4290 RTLIB::ROUND_PPCF128, Results); 4291 break; 4292 case ISD::FROUNDEVEN: 4293 case ISD::STRICT_FROUNDEVEN: 4294 ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32, 4295 RTLIB::ROUNDEVEN_F64, 4296 RTLIB::ROUNDEVEN_F80, 4297 RTLIB::ROUNDEVEN_F128, 4298 RTLIB::ROUNDEVEN_PPCF128, Results); 4299 break; 4300 case ISD::FPOWI: 4301 case ISD::STRICT_FPOWI: { 4302 RTLIB::Libcall LC; 4303 switch (Node->getSimpleValueType(0).SimpleTy) { 4304 default: llvm_unreachable("Unexpected request for libcall!"); 4305 case MVT::f32: LC = RTLIB::POWI_F32; break; 4306 case MVT::f64: LC = RTLIB::POWI_F64; break; 4307 case MVT::f80: LC = RTLIB::POWI_F80; break; 4308 case MVT::f128: LC = RTLIB::POWI_F128; break; 4309 case MVT::ppcf128: LC = RTLIB::POWI_PPCF128; break; 4310 } 4311 if (!TLI.getLibcallName(LC)) { 4312 // Some targets don't have a powi libcall; use pow instead. 4313 SDValue Exponent = DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), 4314 Node->getValueType(0), 4315 Node->getOperand(1)); 4316 Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node), 4317 Node->getValueType(0), Node->getOperand(0), 4318 Exponent)); 4319 break; 4320 } 4321 ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64, 4322 RTLIB::POWI_F80, RTLIB::POWI_F128, 4323 RTLIB::POWI_PPCF128, Results); 4324 break; 4325 } 4326 case ISD::FPOW: 4327 case ISD::STRICT_FPOW: 4328 ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80, 4329 RTLIB::POW_F128, RTLIB::POW_PPCF128, Results); 4330 break; 4331 case ISD::LROUND: 4332 case ISD::STRICT_LROUND: 4333 ExpandArgFPLibCall(Node, RTLIB::LROUND_F32, 4334 RTLIB::LROUND_F64, RTLIB::LROUND_F80, 4335 RTLIB::LROUND_F128, 4336 RTLIB::LROUND_PPCF128, Results); 4337 break; 4338 case ISD::LLROUND: 4339 case ISD::STRICT_LLROUND: 4340 ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32, 4341 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80, 4342 RTLIB::LLROUND_F128, 4343 RTLIB::LLROUND_PPCF128, Results); 4344 break; 4345 case ISD::LRINT: 4346 case ISD::STRICT_LRINT: 4347 ExpandArgFPLibCall(Node, RTLIB::LRINT_F32, 4348 RTLIB::LRINT_F64, RTLIB::LRINT_F80, 4349 RTLIB::LRINT_F128, 4350 RTLIB::LRINT_PPCF128, Results); 4351 break; 4352 case ISD::LLRINT: 4353 case ISD::STRICT_LLRINT: 4354 ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32, 4355 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80, 4356 RTLIB::LLRINT_F128, 4357 RTLIB::LLRINT_PPCF128, Results); 4358 break; 4359 case ISD::FDIV: 4360 case ISD::STRICT_FDIV: 4361 ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64, 4362 RTLIB::DIV_F80, RTLIB::DIV_F128, 4363 RTLIB::DIV_PPCF128, Results); 4364 break; 4365 case ISD::FREM: 4366 case ISD::STRICT_FREM: 4367 ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64, 4368 RTLIB::REM_F80, RTLIB::REM_F128, 4369 RTLIB::REM_PPCF128, Results); 4370 break; 4371 case ISD::FMA: 4372 case ISD::STRICT_FMA: 4373 ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64, 4374 RTLIB::FMA_F80, RTLIB::FMA_F128, 4375 RTLIB::FMA_PPCF128, Results); 4376 break; 4377 case ISD::FADD: 4378 case ISD::STRICT_FADD: 4379 ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64, 4380 RTLIB::ADD_F80, RTLIB::ADD_F128, 4381 RTLIB::ADD_PPCF128, Results); 4382 break; 4383 case ISD::FMUL: 4384 case ISD::STRICT_FMUL: 4385 ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64, 4386 RTLIB::MUL_F80, RTLIB::MUL_F128, 4387 RTLIB::MUL_PPCF128, Results); 4388 break; 4389 case ISD::FP16_TO_FP: 4390 if (Node->getValueType(0) == MVT::f32) { 4391 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false)); 4392 } 4393 break; 4394 case ISD::STRICT_FP16_TO_FP: { 4395 if (Node->getValueType(0) == MVT::f32) { 4396 TargetLowering::MakeLibCallOptions CallOptions; 4397 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall( 4398 DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions, 4399 SDLoc(Node), Node->getOperand(0)); 4400 Results.push_back(Tmp.first); 4401 Results.push_back(Tmp.second); 4402 } 4403 break; 4404 } 4405 case ISD::FP_TO_FP16: { 4406 RTLIB::Libcall LC = 4407 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16); 4408 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16"); 4409 Results.push_back(ExpandLibCall(LC, Node, false)); 4410 break; 4411 } 4412 case ISD::STRICT_SINT_TO_FP: 4413 case ISD::STRICT_UINT_TO_FP: 4414 case ISD::SINT_TO_FP: 4415 case ISD::UINT_TO_FP: { 4416 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP 4417 bool IsStrict = Node->isStrictFPOpcode(); 4418 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP || 4419 Node->getOpcode() == ISD::STRICT_SINT_TO_FP; 4420 EVT SVT = Node->getOperand(IsStrict ? 1 : 0).getValueType(); 4421 EVT RVT = Node->getValueType(0); 4422 EVT NVT = EVT(); 4423 SDLoc dl(Node); 4424 4425 // Even if the input is legal, no libcall may exactly match, eg. we don't 4426 // have i1 -> fp conversions. So, it needs to be promoted to a larger type, 4427 // eg: i13 -> fp. Then, look for an appropriate libcall. 4428 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4429 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE; 4430 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; 4431 ++t) { 4432 NVT = (MVT::SimpleValueType)t; 4433 // The source needs to big enough to hold the operand. 4434 if (NVT.bitsGE(SVT)) 4435 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT) 4436 : RTLIB::getUINTTOFP(NVT, RVT); 4437 } 4438 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall"); 4439 4440 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); 4441 // Sign/zero extend the argument if the libcall takes a larger type. 4442 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, 4443 NVT, Node->getOperand(IsStrict ? 1 : 0)); 4444 TargetLowering::MakeLibCallOptions CallOptions; 4445 CallOptions.setSExt(Signed); 4446 std::pair<SDValue, SDValue> Tmp = 4447 TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, dl, Chain); 4448 Results.push_back(Tmp.first); 4449 if (IsStrict) 4450 Results.push_back(Tmp.second); 4451 break; 4452 } 4453 case ISD::FP_TO_SINT: 4454 case ISD::FP_TO_UINT: 4455 case ISD::STRICT_FP_TO_SINT: 4456 case ISD::STRICT_FP_TO_UINT: { 4457 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT. 4458 bool IsStrict = Node->isStrictFPOpcode(); 4459 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT || 4460 Node->getOpcode() == ISD::STRICT_FP_TO_SINT; 4461 4462 SDValue Op = Node->getOperand(IsStrict ? 1 : 0); 4463 EVT SVT = Op.getValueType(); 4464 EVT RVT = Node->getValueType(0); 4465 EVT NVT = EVT(); 4466 SDLoc dl(Node); 4467 4468 // Even if the result is legal, no libcall may exactly match, eg. we don't 4469 // have fp -> i1 conversions. So, it needs to be promoted to a larger type, 4470 // eg: fp -> i32. Then, look for an appropriate libcall. 4471 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4472 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE; 4473 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; 4474 ++IntVT) { 4475 NVT = (MVT::SimpleValueType)IntVT; 4476 // The type needs to big enough to hold the result. 4477 if (NVT.bitsGE(RVT)) 4478 LC = Signed ? RTLIB::getFPTOSINT(SVT, NVT) 4479 : RTLIB::getFPTOUINT(SVT, NVT); 4480 } 4481 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall"); 4482 4483 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); 4484 TargetLowering::MakeLibCallOptions CallOptions; 4485 std::pair<SDValue, SDValue> Tmp = 4486 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain); 4487 4488 // Truncate the result if the libcall returns a larger type. 4489 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first)); 4490 if (IsStrict) 4491 Results.push_back(Tmp.second); 4492 break; 4493 } 4494 4495 case ISD::FP_ROUND: 4496 case ISD::STRICT_FP_ROUND: { 4497 // X = FP_ROUND(Y, TRUNC) 4498 // TRUNC is a flag, which is always an integer that is zero or one. 4499 // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND 4500 // is known to not change the value of Y. 4501 // We can only expand it into libcall if the TRUNC is 0. 4502 bool IsStrict = Node->isStrictFPOpcode(); 4503 SDValue Op = Node->getOperand(IsStrict ? 1 : 0); 4504 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); 4505 EVT VT = Node->getValueType(0); 4506 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1)) 4507 ->isNullValue() && 4508 "Unable to expand as libcall if it is not normal rounding"); 4509 4510 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), VT); 4511 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall"); 4512 4513 TargetLowering::MakeLibCallOptions CallOptions; 4514 std::pair<SDValue, SDValue> Tmp = 4515 TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, SDLoc(Node), Chain); 4516 Results.push_back(Tmp.first); 4517 if (IsStrict) 4518 Results.push_back(Tmp.second); 4519 break; 4520 } 4521 case ISD::FP_EXTEND: { 4522 Results.push_back( 4523 ExpandLibCall(RTLIB::getFPEXT(Node->getOperand(0).getValueType(), 4524 Node->getValueType(0)), 4525 Node, false)); 4526 break; 4527 } 4528 case ISD::STRICT_FP_EXTEND: 4529 case ISD::STRICT_FP_TO_FP16: { 4530 RTLIB::Libcall LC = 4531 Node->getOpcode() == ISD::STRICT_FP_TO_FP16 4532 ? RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16) 4533 : RTLIB::getFPEXT(Node->getOperand(1).getValueType(), 4534 Node->getValueType(0)); 4535 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall"); 4536 4537 TargetLowering::MakeLibCallOptions CallOptions; 4538 std::pair<SDValue, SDValue> Tmp = 4539 TLI.makeLibCall(DAG, LC, Node->getValueType(0), Node->getOperand(1), 4540 CallOptions, SDLoc(Node), Node->getOperand(0)); 4541 Results.push_back(Tmp.first); 4542 Results.push_back(Tmp.second); 4543 break; 4544 } 4545 case ISD::FSUB: 4546 case ISD::STRICT_FSUB: 4547 ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64, 4548 RTLIB::SUB_F80, RTLIB::SUB_F128, 4549 RTLIB::SUB_PPCF128, Results); 4550 break; 4551 case ISD::SREM: 4552 Results.push_back(ExpandIntLibCall(Node, true, 4553 RTLIB::SREM_I8, 4554 RTLIB::SREM_I16, RTLIB::SREM_I32, 4555 RTLIB::SREM_I64, RTLIB::SREM_I128)); 4556 break; 4557 case ISD::UREM: 4558 Results.push_back(ExpandIntLibCall(Node, false, 4559 RTLIB::UREM_I8, 4560 RTLIB::UREM_I16, RTLIB::UREM_I32, 4561 RTLIB::UREM_I64, RTLIB::UREM_I128)); 4562 break; 4563 case ISD::SDIV: 4564 Results.push_back(ExpandIntLibCall(Node, true, 4565 RTLIB::SDIV_I8, 4566 RTLIB::SDIV_I16, RTLIB::SDIV_I32, 4567 RTLIB::SDIV_I64, RTLIB::SDIV_I128)); 4568 break; 4569 case ISD::UDIV: 4570 Results.push_back(ExpandIntLibCall(Node, false, 4571 RTLIB::UDIV_I8, 4572 RTLIB::UDIV_I16, RTLIB::UDIV_I32, 4573 RTLIB::UDIV_I64, RTLIB::UDIV_I128)); 4574 break; 4575 case ISD::SDIVREM: 4576 case ISD::UDIVREM: 4577 // Expand into divrem libcall 4578 ExpandDivRemLibCall(Node, Results); 4579 break; 4580 case ISD::MUL: 4581 Results.push_back(ExpandIntLibCall(Node, false, 4582 RTLIB::MUL_I8, 4583 RTLIB::MUL_I16, RTLIB::MUL_I32, 4584 RTLIB::MUL_I64, RTLIB::MUL_I128)); 4585 break; 4586 case ISD::CTLZ_ZERO_UNDEF: 4587 switch (Node->getSimpleValueType(0).SimpleTy) { 4588 default: 4589 llvm_unreachable("LibCall explicitly requested, but not available"); 4590 case MVT::i32: 4591 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false)); 4592 break; 4593 case MVT::i64: 4594 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false)); 4595 break; 4596 case MVT::i128: 4597 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false)); 4598 break; 4599 } 4600 break; 4601 } 4602 4603 // Replace the original node with the legalized result. 4604 if (!Results.empty()) { 4605 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n"); 4606 ReplaceNode(Node, Results.data()); 4607 } else 4608 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n"); 4609 } 4610 4611 // Determine the vector type to use in place of an original scalar element when 4612 // promoting equally sized vectors. 4613 static MVT getPromotedVectorElementType(const TargetLowering &TLI, 4614 MVT EltVT, MVT NewEltVT) { 4615 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits(); 4616 MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt); 4617 assert(TLI.isTypeLegal(MidVT) && "unexpected"); 4618 return MidVT; 4619 } 4620 4621 void SelectionDAGLegalize::PromoteNode(SDNode *Node) { 4622 LLVM_DEBUG(dbgs() << "Trying to promote node\n"); 4623 SmallVector<SDValue, 8> Results; 4624 MVT OVT = Node->getSimpleValueType(0); 4625 if (Node->getOpcode() == ISD::UINT_TO_FP || 4626 Node->getOpcode() == ISD::SINT_TO_FP || 4627 Node->getOpcode() == ISD::SETCC || 4628 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT || 4629 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) { 4630 OVT = Node->getOperand(0).getSimpleValueType(); 4631 } 4632 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP || 4633 Node->getOpcode() == ISD::STRICT_SINT_TO_FP || 4634 Node->getOpcode() == ISD::STRICT_FSETCC || 4635 Node->getOpcode() == ISD::STRICT_FSETCCS) 4636 OVT = Node->getOperand(1).getSimpleValueType(); 4637 if (Node->getOpcode() == ISD::BR_CC) 4638 OVT = Node->getOperand(2).getSimpleValueType(); 4639 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); 4640 SDLoc dl(Node); 4641 SDValue Tmp1, Tmp2, Tmp3; 4642 switch (Node->getOpcode()) { 4643 case ISD::CTTZ: 4644 case ISD::CTTZ_ZERO_UNDEF: 4645 case ISD::CTLZ: 4646 case ISD::CTLZ_ZERO_UNDEF: 4647 case ISD::CTPOP: 4648 // Zero extend the argument unless its cttz, then use any_extend. 4649 if (Node->getOpcode() == ISD::CTTZ || 4650 Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF) 4651 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); 4652 else 4653 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); 4654 4655 if (Node->getOpcode() == ISD::CTTZ) { 4656 // The count is the same in the promoted type except if the original 4657 // value was zero. This can be handled by setting the bit just off 4658 // the top of the original type. 4659 auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(), 4660 OVT.getSizeInBits()); 4661 Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1, 4662 DAG.getConstant(TopBit, dl, NVT)); 4663 } 4664 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is 4665 // already the correct result. 4666 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 4667 if (Node->getOpcode() == ISD::CTLZ || 4668 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) { 4669 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT)) 4670 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, 4671 DAG.getConstant(NVT.getSizeInBits() - 4672 OVT.getSizeInBits(), dl, NVT)); 4673 } 4674 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); 4675 break; 4676 case ISD::BITREVERSE: 4677 case ISD::BSWAP: { 4678 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); 4679 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); 4680 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 4681 Tmp1 = DAG.getNode( 4682 ISD::SRL, dl, NVT, Tmp1, 4683 DAG.getConstant(DiffBits, dl, 4684 TLI.getShiftAmountTy(NVT, DAG.getDataLayout()))); 4685 4686 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); 4687 break; 4688 } 4689 case ISD::FP_TO_UINT: 4690 case ISD::STRICT_FP_TO_UINT: 4691 case ISD::FP_TO_SINT: 4692 case ISD::STRICT_FP_TO_SINT: 4693 PromoteLegalFP_TO_INT(Node, dl, Results); 4694 break; 4695 case ISD::FP_TO_UINT_SAT: 4696 case ISD::FP_TO_SINT_SAT: 4697 Results.push_back(PromoteLegalFP_TO_INT_SAT(Node, dl)); 4698 break; 4699 case ISD::UINT_TO_FP: 4700 case ISD::STRICT_UINT_TO_FP: 4701 case ISD::SINT_TO_FP: 4702 case ISD::STRICT_SINT_TO_FP: 4703 PromoteLegalINT_TO_FP(Node, dl, Results); 4704 break; 4705 case ISD::VAARG: { 4706 SDValue Chain = Node->getOperand(0); // Get the chain. 4707 SDValue Ptr = Node->getOperand(1); // Get the pointer. 4708 4709 unsigned TruncOp; 4710 if (OVT.isVector()) { 4711 TruncOp = ISD::BITCAST; 4712 } else { 4713 assert(OVT.isInteger() 4714 && "VAARG promotion is supported only for vectors or integer types"); 4715 TruncOp = ISD::TRUNCATE; 4716 } 4717 4718 // Perform the larger operation, then convert back 4719 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2), 4720 Node->getConstantOperandVal(3)); 4721 Chain = Tmp1.getValue(1); 4722 4723 Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1); 4724 4725 // Modified the chain result - switch anything that used the old chain to 4726 // use the new one. 4727 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2); 4728 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); 4729 if (UpdatedNodes) { 4730 UpdatedNodes->insert(Tmp2.getNode()); 4731 UpdatedNodes->insert(Chain.getNode()); 4732 } 4733 ReplacedNode(Node); 4734 break; 4735 } 4736 case ISD::MUL: 4737 case ISD::SDIV: 4738 case ISD::SREM: 4739 case ISD::UDIV: 4740 case ISD::UREM: 4741 case ISD::AND: 4742 case ISD::OR: 4743 case ISD::XOR: { 4744 unsigned ExtOp, TruncOp; 4745 if (OVT.isVector()) { 4746 ExtOp = ISD::BITCAST; 4747 TruncOp = ISD::BITCAST; 4748 } else { 4749 assert(OVT.isInteger() && "Cannot promote logic operation"); 4750 4751 switch (Node->getOpcode()) { 4752 default: 4753 ExtOp = ISD::ANY_EXTEND; 4754 break; 4755 case ISD::SDIV: 4756 case ISD::SREM: 4757 ExtOp = ISD::SIGN_EXTEND; 4758 break; 4759 case ISD::UDIV: 4760 case ISD::UREM: 4761 ExtOp = ISD::ZERO_EXTEND; 4762 break; 4763 } 4764 TruncOp = ISD::TRUNCATE; 4765 } 4766 // Promote each of the values to the new type. 4767 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); 4768 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 4769 // Perform the larger operation, then convert back 4770 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 4771 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1)); 4772 break; 4773 } 4774 case ISD::UMUL_LOHI: 4775 case ISD::SMUL_LOHI: { 4776 // Promote to a multiply in a wider integer type. 4777 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND 4778 : ISD::SIGN_EXTEND; 4779 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); 4780 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 4781 Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2); 4782 4783 auto &DL = DAG.getDataLayout(); 4784 unsigned OriginalSize = OVT.getScalarSizeInBits(); 4785 Tmp2 = DAG.getNode( 4786 ISD::SRL, dl, NVT, Tmp1, 4787 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT))); 4788 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); 4789 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2)); 4790 break; 4791 } 4792 case ISD::SELECT: { 4793 unsigned ExtOp, TruncOp; 4794 if (Node->getValueType(0).isVector() || 4795 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) { 4796 ExtOp = ISD::BITCAST; 4797 TruncOp = ISD::BITCAST; 4798 } else if (Node->getValueType(0).isInteger()) { 4799 ExtOp = ISD::ANY_EXTEND; 4800 TruncOp = ISD::TRUNCATE; 4801 } else { 4802 ExtOp = ISD::FP_EXTEND; 4803 TruncOp = ISD::FP_ROUND; 4804 } 4805 Tmp1 = Node->getOperand(0); 4806 // Promote each of the values to the new type. 4807 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 4808 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); 4809 // Perform the larger operation, then round down. 4810 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3); 4811 Tmp1->setFlags(Node->getFlags()); 4812 if (TruncOp != ISD::FP_ROUND) 4813 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1); 4814 else 4815 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1, 4816 DAG.getIntPtrConstant(0, dl)); 4817 Results.push_back(Tmp1); 4818 break; 4819 } 4820 case ISD::VECTOR_SHUFFLE: { 4821 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); 4822 4823 // Cast the two input vectors. 4824 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); 4825 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); 4826 4827 // Convert the shuffle mask to the right # elements. 4828 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask); 4829 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1); 4830 Results.push_back(Tmp1); 4831 break; 4832 } 4833 case ISD::SETCC: 4834 case ISD::STRICT_FSETCC: 4835 case ISD::STRICT_FSETCCS: { 4836 unsigned ExtOp = ISD::FP_EXTEND; 4837 if (NVT.isInteger()) { 4838 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); 4839 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 4840 } 4841 if (Node->isStrictFPOpcode()) { 4842 SDValue InChain = Node->getOperand(0); 4843 std::tie(Tmp1, std::ignore) = 4844 DAG.getStrictFPExtendOrRound(Node->getOperand(1), InChain, dl, NVT); 4845 std::tie(Tmp2, std::ignore) = 4846 DAG.getStrictFPExtendOrRound(Node->getOperand(2), InChain, dl, NVT); 4847 SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(1), Tmp2.getValue(1)}; 4848 SDValue OutChain = DAG.getTokenFactor(dl, TmpChains); 4849 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); 4850 Results.push_back(DAG.getNode(Node->getOpcode(), dl, VTs, 4851 {OutChain, Tmp1, Tmp2, Node->getOperand(3)}, 4852 Node->getFlags())); 4853 Results.push_back(Results.back().getValue(1)); 4854 break; 4855 } 4856 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); 4857 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 4858 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1, 4859 Tmp2, Node->getOperand(2), Node->getFlags())); 4860 break; 4861 } 4862 case ISD::BR_CC: { 4863 unsigned ExtOp = ISD::FP_EXTEND; 4864 if (NVT.isInteger()) { 4865 ISD::CondCode CCCode = 4866 cast<CondCodeSDNode>(Node->getOperand(1))->get(); 4867 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 4868 } 4869 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); 4870 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); 4871 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), 4872 Node->getOperand(0), Node->getOperand(1), 4873 Tmp1, Tmp2, Node->getOperand(4))); 4874 break; 4875 } 4876 case ISD::FADD: 4877 case ISD::FSUB: 4878 case ISD::FMUL: 4879 case ISD::FDIV: 4880 case ISD::FREM: 4881 case ISD::FMINNUM: 4882 case ISD::FMAXNUM: 4883 case ISD::FPOW: 4884 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); 4885 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); 4886 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, 4887 Node->getFlags()); 4888 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, 4889 Tmp3, DAG.getIntPtrConstant(0, dl))); 4890 break; 4891 case ISD::STRICT_FREM: 4892 case ISD::STRICT_FPOW: 4893 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, 4894 {Node->getOperand(0), Node->getOperand(1)}); 4895 Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, 4896 {Node->getOperand(0), Node->getOperand(2)}); 4897 Tmp3 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1), 4898 Tmp2.getValue(1)); 4899 Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, 4900 {Tmp3, Tmp1, Tmp2}); 4901 Tmp1 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, 4902 {Tmp1.getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)}); 4903 Results.push_back(Tmp1); 4904 Results.push_back(Tmp1.getValue(1)); 4905 break; 4906 case ISD::FMA: 4907 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); 4908 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); 4909 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2)); 4910 Results.push_back( 4911 DAG.getNode(ISD::FP_ROUND, dl, OVT, 4912 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3), 4913 DAG.getIntPtrConstant(0, dl))); 4914 break; 4915 case ISD::FCOPYSIGN: 4916 case ISD::FPOWI: { 4917 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); 4918 Tmp2 = Node->getOperand(1); 4919 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 4920 4921 // fcopysign doesn't change anything but the sign bit, so 4922 // (fp_round (fcopysign (fpext a), b)) 4923 // is as precise as 4924 // (fp_round (fpext a)) 4925 // which is a no-op. Mark it as a TRUNCating FP_ROUND. 4926 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN); 4927 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, 4928 Tmp3, DAG.getIntPtrConstant(isTrunc, dl))); 4929 break; 4930 } 4931 case ISD::FFLOOR: 4932 case ISD::FCEIL: 4933 case ISD::FRINT: 4934 case ISD::FNEARBYINT: 4935 case ISD::FROUND: 4936 case ISD::FROUNDEVEN: 4937 case ISD::FTRUNC: 4938 case ISD::FNEG: 4939 case ISD::FSQRT: 4940 case ISD::FSIN: 4941 case ISD::FCOS: 4942 case ISD::FLOG: 4943 case ISD::FLOG2: 4944 case ISD::FLOG10: 4945 case ISD::FABS: 4946 case ISD::FEXP: 4947 case ISD::FEXP2: 4948 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); 4949 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 4950 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, 4951 Tmp2, DAG.getIntPtrConstant(0, dl))); 4952 break; 4953 case ISD::STRICT_FFLOOR: 4954 case ISD::STRICT_FCEIL: 4955 case ISD::STRICT_FSIN: 4956 case ISD::STRICT_FCOS: 4957 case ISD::STRICT_FLOG: 4958 case ISD::STRICT_FLOG10: 4959 case ISD::STRICT_FEXP: 4960 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, 4961 {Node->getOperand(0), Node->getOperand(1)}); 4962 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, 4963 {Tmp1.getValue(1), Tmp1}); 4964 Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, 4965 {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)}); 4966 Results.push_back(Tmp3); 4967 Results.push_back(Tmp3.getValue(1)); 4968 break; 4969 case ISD::BUILD_VECTOR: { 4970 MVT EltVT = OVT.getVectorElementType(); 4971 MVT NewEltVT = NVT.getVectorElementType(); 4972 4973 // Handle bitcasts to a different vector type with the same total bit size 4974 // 4975 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32 4976 // => 4977 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y)) 4978 4979 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && 4980 "Invalid promote type for build_vector"); 4981 assert(NewEltVT.bitsLT(EltVT) && "not handled"); 4982 4983 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); 4984 4985 SmallVector<SDValue, 8> NewOps; 4986 for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) { 4987 SDValue Op = Node->getOperand(I); 4988 NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op)); 4989 } 4990 4991 SDLoc SL(Node); 4992 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps); 4993 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); 4994 Results.push_back(CvtVec); 4995 break; 4996 } 4997 case ISD::EXTRACT_VECTOR_ELT: { 4998 MVT EltVT = OVT.getVectorElementType(); 4999 MVT NewEltVT = NVT.getVectorElementType(); 5000 5001 // Handle bitcasts to a different vector type with the same total bit size. 5002 // 5003 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32 5004 // => 5005 // v4i32:castx = bitcast x:v2i64 5006 // 5007 // i64 = bitcast 5008 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))), 5009 // (i32 (extract_vector_elt castx, (2 * y + 1))) 5010 // 5011 5012 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && 5013 "Invalid promote type for extract_vector_elt"); 5014 assert(NewEltVT.bitsLT(EltVT) && "not handled"); 5015 5016 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); 5017 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements(); 5018 5019 SDValue Idx = Node->getOperand(1); 5020 EVT IdxVT = Idx.getValueType(); 5021 SDLoc SL(Node); 5022 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT); 5023 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); 5024 5025 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); 5026 5027 SmallVector<SDValue, 8> NewOps; 5028 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) { 5029 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT); 5030 SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); 5031 5032 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, 5033 CastVec, TmpIdx); 5034 NewOps.push_back(Elt); 5035 } 5036 5037 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps); 5038 Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec)); 5039 break; 5040 } 5041 case ISD::INSERT_VECTOR_ELT: { 5042 MVT EltVT = OVT.getVectorElementType(); 5043 MVT NewEltVT = NVT.getVectorElementType(); 5044 5045 // Handle bitcasts to a different vector type with the same total bit size 5046 // 5047 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32 5048 // => 5049 // v4i32:castx = bitcast x:v2i64 5050 // v2i32:casty = bitcast y:i64 5051 // 5052 // v2i64 = bitcast 5053 // (v4i32 insert_vector_elt 5054 // (v4i32 insert_vector_elt v4i32:castx, 5055 // (extract_vector_elt casty, 0), 2 * z), 5056 // (extract_vector_elt casty, 1), (2 * z + 1)) 5057 5058 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && 5059 "Invalid promote type for insert_vector_elt"); 5060 assert(NewEltVT.bitsLT(EltVT) && "not handled"); 5061 5062 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); 5063 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements(); 5064 5065 SDValue Val = Node->getOperand(1); 5066 SDValue Idx = Node->getOperand(2); 5067 EVT IdxVT = Idx.getValueType(); 5068 SDLoc SL(Node); 5069 5070 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT); 5071 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); 5072 5073 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); 5074 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); 5075 5076 SDValue NewVec = CastVec; 5077 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) { 5078 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT); 5079 SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); 5080 5081 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, 5082 CastVal, IdxOffset); 5083 5084 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT, 5085 NewVec, Elt, InEltIdx); 5086 } 5087 5088 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec)); 5089 break; 5090 } 5091 case ISD::SCALAR_TO_VECTOR: { 5092 MVT EltVT = OVT.getVectorElementType(); 5093 MVT NewEltVT = NVT.getVectorElementType(); 5094 5095 // Handle bitcasts to different vector type with the same total bit size. 5096 // 5097 // e.g. v2i64 = scalar_to_vector x:i64 5098 // => 5099 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef) 5100 // 5101 5102 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); 5103 SDValue Val = Node->getOperand(0); 5104 SDLoc SL(Node); 5105 5106 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); 5107 SDValue Undef = DAG.getUNDEF(MidVT); 5108 5109 SmallVector<SDValue, 8> NewElts; 5110 NewElts.push_back(CastVal); 5111 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I) 5112 NewElts.push_back(Undef); 5113 5114 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts); 5115 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); 5116 Results.push_back(CvtVec); 5117 break; 5118 } 5119 case ISD::ATOMIC_SWAP: { 5120 AtomicSDNode *AM = cast<AtomicSDNode>(Node); 5121 SDLoc SL(Node); 5122 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal()); 5123 assert(NVT.getSizeInBits() == OVT.getSizeInBits() && 5124 "unexpected promotion type"); 5125 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() && 5126 "unexpected atomic_swap with illegal type"); 5127 5128 SDValue NewAtomic 5129 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT, 5130 DAG.getVTList(NVT, MVT::Other), 5131 { AM->getChain(), AM->getBasePtr(), CastVal }, 5132 AM->getMemOperand()); 5133 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic)); 5134 Results.push_back(NewAtomic.getValue(1)); 5135 break; 5136 } 5137 } 5138 5139 // Replace the original node with the legalized result. 5140 if (!Results.empty()) { 5141 LLVM_DEBUG(dbgs() << "Successfully promoted node\n"); 5142 ReplaceNode(Node, Results.data()); 5143 } else 5144 LLVM_DEBUG(dbgs() << "Could not promote node\n"); 5145 } 5146 5147 /// This is the entry point for the file. 5148 void SelectionDAG::Legalize() { 5149 AssignTopologicalOrder(); 5150 5151 SmallPtrSet<SDNode *, 16> LegalizedNodes; 5152 // Use a delete listener to remove nodes which were deleted during 5153 // legalization from LegalizeNodes. This is needed to handle the situation 5154 // where a new node is allocated by the object pool to the same address of a 5155 // previously deleted node. 5156 DAGNodeDeletedListener DeleteListener( 5157 *this, 5158 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); }); 5159 5160 SelectionDAGLegalize Legalizer(*this, LegalizedNodes); 5161 5162 // Visit all the nodes. We start in topological order, so that we see 5163 // nodes with their original operands intact. Legalization can produce 5164 // new nodes which may themselves need to be legalized. Iterate until all 5165 // nodes have been legalized. 5166 while (true) { 5167 bool AnyLegalized = false; 5168 for (auto NI = allnodes_end(); NI != allnodes_begin();) { 5169 --NI; 5170 5171 SDNode *N = &*NI; 5172 if (N->use_empty() && N != getRoot().getNode()) { 5173 ++NI; 5174 DeleteNode(N); 5175 continue; 5176 } 5177 5178 if (LegalizedNodes.insert(N).second) { 5179 AnyLegalized = true; 5180 Legalizer.LegalizeOp(N); 5181 5182 if (N->use_empty() && N != getRoot().getNode()) { 5183 ++NI; 5184 DeleteNode(N); 5185 } 5186 } 5187 } 5188 if (!AnyLegalized) 5189 break; 5190 5191 } 5192 5193 // Remove dead nodes now. 5194 RemoveDeadNodes(); 5195 } 5196 5197 bool SelectionDAG::LegalizeOp(SDNode *N, 5198 SmallSetVector<SDNode *, 16> &UpdatedNodes) { 5199 SmallPtrSet<SDNode *, 16> LegalizedNodes; 5200 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes); 5201 5202 // Directly insert the node in question, and legalize it. This will recurse 5203 // as needed through operands. 5204 LegalizedNodes.insert(N); 5205 Legalizer.LegalizeOp(N); 5206 5207 return LegalizedNodes.count(N); 5208 } 5209