1 //===-------- LegalizeTypesGeneric.cpp - Generic type legalization --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements generic type expansion and splitting for LegalizeTypes. 10 // The routines here perform legalization when the details of the type (such as 11 // whether it is an integer or a float) do not matter. 12 // Expansion is the act of changing a computation in an illegal type to be a 13 // computation in two identical registers of a smaller type. The Lo/Hi part 14 // is required to be stored first in memory on little/big-endian machines. 15 // Splitting is the act of changing a computation in an illegal type to be a 16 // computation in two not necessarily identical registers of a smaller type. 17 // There are no requirements on how the type is represented in memory. 18 // 19 //===----------------------------------------------------------------------===// 20 21 #include "LegalizeTypes.h" 22 #include "llvm/IR/DataLayout.h" 23 using namespace llvm; 24 25 #define DEBUG_TYPE "legalize-types" 26 27 //===----------------------------------------------------------------------===// 28 // Generic Result Expansion. 29 //===----------------------------------------------------------------------===// 30 31 // These routines assume that the Lo/Hi part is stored first in memory on 32 // little/big-endian machines, followed by the Hi/Lo part. This means that 33 // they cannot be used as is on vectors, for which Lo is always stored first. 34 void DAGTypeLegalizer::ExpandRes_MERGE_VALUES(SDNode *N, unsigned ResNo, 35 SDValue &Lo, SDValue &Hi) { 36 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 37 GetExpandedOp(Op, Lo, Hi); 38 } 39 40 void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) { 41 EVT OutVT = N->getValueType(0); 42 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 43 SDValue InOp = N->getOperand(0); 44 EVT InVT = InOp.getValueType(); 45 SDLoc dl(N); 46 47 // Handle some special cases efficiently. 48 switch (getTypeAction(InVT)) { 49 case TargetLowering::TypeLegal: 50 case TargetLowering::TypePromoteInteger: 51 break; 52 case TargetLowering::TypePromoteFloat: 53 case TargetLowering::TypeSoftPromoteHalf: 54 llvm_unreachable("Bitcast of a promotion-needing float should never need" 55 "expansion"); 56 case TargetLowering::TypeSoftenFloat: 57 SplitInteger(GetSoftenedFloat(InOp), Lo, Hi); 58 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 59 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 60 return; 61 case TargetLowering::TypeExpandInteger: 62 case TargetLowering::TypeExpandFloat: { 63 auto &DL = DAG.getDataLayout(); 64 // Convert the expanded pieces of the input. 65 GetExpandedOp(InOp, Lo, Hi); 66 if (TLI.hasBigEndianPartOrdering(InVT, DL) != 67 TLI.hasBigEndianPartOrdering(OutVT, DL)) 68 std::swap(Lo, Hi); 69 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 70 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 71 return; 72 } 73 case TargetLowering::TypeSplitVector: 74 GetSplitVector(InOp, Lo, Hi); 75 if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout())) 76 std::swap(Lo, Hi); 77 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 78 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 79 return; 80 case TargetLowering::TypeScalarizeVector: 81 // Convert the element instead. 82 SplitInteger(BitConvertToInteger(GetScalarizedVector(InOp)), Lo, Hi); 83 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 84 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 85 return; 86 case TargetLowering::TypeScalarizeScalableVector: 87 report_fatal_error("Scalarization of scalable vectors is not supported."); 88 case TargetLowering::TypeWidenVector: { 89 assert(!(InVT.getVectorNumElements() & 1) && "Unsupported BITCAST"); 90 InOp = GetWidenedVector(InOp); 91 EVT LoVT, HiVT; 92 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(InVT); 93 std::tie(Lo, Hi) = DAG.SplitVector(InOp, dl, LoVT, HiVT); 94 if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout())) 95 std::swap(Lo, Hi); 96 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 97 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 98 return; 99 } 100 } 101 102 if (InVT.isVector() && OutVT.isInteger()) { 103 // Handle cases like i64 = BITCAST v1i64 on x86, where the operand 104 // is legal but the result is not. 105 unsigned NumElems = 2; 106 EVT ElemVT = NOutVT; 107 EVT NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems); 108 109 // If <ElemVT * N> is not a legal type, try <ElemVT/2 * (N*2)>. 110 while (!isTypeLegal(NVT)) { 111 unsigned NewSizeInBits = ElemVT.getSizeInBits() / 2; 112 // If the element size is smaller than byte, bail. 113 if (NewSizeInBits < 8) 114 break; 115 NumElems *= 2; 116 ElemVT = EVT::getIntegerVT(*DAG.getContext(), NewSizeInBits); 117 NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems); 118 } 119 120 if (isTypeLegal(NVT)) { 121 SDValue CastInOp = DAG.getNode(ISD::BITCAST, dl, NVT, InOp); 122 123 SmallVector<SDValue, 8> Vals; 124 for (unsigned i = 0; i < NumElems; ++i) 125 Vals.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ElemVT, 126 CastInOp, DAG.getVectorIdxConstant(i, dl))); 127 128 // Build Lo, Hi pair by pairing extracted elements if needed. 129 unsigned Slot = 0; 130 for (unsigned e = Vals.size(); e - Slot > 2; Slot += 2, e += 1) { 131 // Each iteration will BUILD_PAIR two nodes and append the result until 132 // there are only two nodes left, i.e. Lo and Hi. 133 SDValue LHS = Vals[Slot]; 134 SDValue RHS = Vals[Slot + 1]; 135 136 if (DAG.getDataLayout().isBigEndian()) 137 std::swap(LHS, RHS); 138 139 Vals.push_back(DAG.getNode( 140 ISD::BUILD_PAIR, dl, 141 EVT::getIntegerVT(*DAG.getContext(), LHS.getValueSizeInBits() << 1), 142 LHS, RHS)); 143 } 144 Lo = Vals[Slot++]; 145 Hi = Vals[Slot++]; 146 147 if (DAG.getDataLayout().isBigEndian()) 148 std::swap(Lo, Hi); 149 150 return; 151 } 152 } 153 154 // Lower the bit-convert to a store/load from the stack. 155 assert(NOutVT.isByteSized() && "Expanded type not byte sized!"); 156 157 // Create the stack frame object. Make sure it is aligned for both 158 // the source and expanded destination types. 159 160 // In cases where the vector is illegal it will be broken down into parts 161 // and stored in parts - we should use the alignment for the smallest part. 162 Align InAlign = DAG.getReducedAlign(InVT, /*UseABI=*/false); 163 Align NOutAlign = DAG.getReducedAlign(NOutVT, /*UseABI=*/false); 164 Align Align = std::max(InAlign, NOutAlign); 165 SDValue StackPtr = DAG.CreateStackTemporary(InVT.getStoreSize(), Align); 166 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 167 MachinePointerInfo PtrInfo = 168 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); 169 170 // Emit a store to the stack slot. 171 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, PtrInfo); 172 173 // Load the first half from the stack slot. 174 Lo = DAG.getLoad(NOutVT, dl, Store, StackPtr, PtrInfo, NOutAlign); 175 176 // Increment the pointer to the other half. 177 unsigned IncrementSize = NOutVT.getSizeInBits() / 8; 178 StackPtr = 179 DAG.getMemBasePlusOffset(StackPtr, TypeSize::getFixed(IncrementSize), dl); 180 181 // Load the second half from the stack slot. 182 Hi = DAG.getLoad(NOutVT, dl, Store, StackPtr, 183 PtrInfo.getWithOffset(IncrementSize), NOutAlign); 184 185 // Handle endianness of the load. 186 if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout())) 187 std::swap(Lo, Hi); 188 } 189 190 void DAGTypeLegalizer::ExpandRes_BUILD_PAIR(SDNode *N, SDValue &Lo, 191 SDValue &Hi) { 192 // Return the operands. 193 Lo = N->getOperand(0); 194 Hi = N->getOperand(1); 195 } 196 197 void DAGTypeLegalizer::ExpandRes_EXTRACT_ELEMENT(SDNode *N, SDValue &Lo, 198 SDValue &Hi) { 199 GetExpandedOp(N->getOperand(0), Lo, Hi); 200 SDValue Part = N->getConstantOperandVal(1) ? Hi : Lo; 201 202 assert(Part.getValueType() == N->getValueType(0) && 203 "Type twice as big as expanded type not itself expanded!"); 204 205 GetPairElements(Part, Lo, Hi); 206 } 207 208 void DAGTypeLegalizer::ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo, 209 SDValue &Hi) { 210 SDValue OldVec = N->getOperand(0); 211 ElementCount OldEltCount = OldVec.getValueType().getVectorElementCount(); 212 EVT OldEltVT = OldVec.getValueType().getVectorElementType(); 213 SDLoc dl(N); 214 215 // Convert to a vector of the expanded element type, for example 216 // <3 x i64> -> <6 x i32>. 217 EVT OldVT = N->getValueType(0); 218 EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT); 219 220 if (OldVT != OldEltVT) { 221 // The result of EXTRACT_VECTOR_ELT may be larger than the element type of 222 // the input vector. If so, extend the elements of the input vector to the 223 // same bitwidth as the result before expanding. 224 assert(OldEltVT.bitsLT(OldVT) && "Result type smaller then element type!"); 225 EVT NVecVT = EVT::getVectorVT(*DAG.getContext(), OldVT, OldEltCount); 226 OldVec = DAG.getNode(ISD::ANY_EXTEND, dl, NVecVT, N->getOperand(0)); 227 } 228 229 SDValue NewVec = DAG.getNode( 230 ISD::BITCAST, dl, 231 EVT::getVectorVT(*DAG.getContext(), NewVT, OldEltCount * 2), OldVec); 232 233 // Extract the elements at 2 * Idx and 2 * Idx + 1 from the new vector. 234 SDValue Idx = N->getOperand(1); 235 236 Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx); 237 Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx); 238 239 Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, 240 DAG.getConstant(1, dl, Idx.getValueType())); 241 Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx); 242 243 if (DAG.getDataLayout().isBigEndian()) 244 std::swap(Lo, Hi); 245 } 246 247 void DAGTypeLegalizer::ExpandRes_NormalLoad(SDNode *N, SDValue &Lo, 248 SDValue &Hi) { 249 assert(ISD::isNormalLoad(N) && "This routine only for normal loads!"); 250 SDLoc dl(N); 251 252 LoadSDNode *LD = cast<LoadSDNode>(N); 253 assert(!LD->isAtomic() && "Atomics can not be split"); 254 EVT ValueVT = LD->getValueType(0); 255 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT); 256 SDValue Chain = LD->getChain(); 257 SDValue Ptr = LD->getBasePtr(); 258 AAMDNodes AAInfo = LD->getAAInfo(); 259 260 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 261 262 Lo = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getPointerInfo(), 263 LD->getOriginalAlign(), LD->getMemOperand()->getFlags(), 264 AAInfo); 265 266 // Increment the pointer to the other half. 267 unsigned IncrementSize = NVT.getSizeInBits() / 8; 268 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl); 269 Hi = DAG.getLoad( 270 NVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(IncrementSize), 271 LD->getOriginalAlign(), LD->getMemOperand()->getFlags(), AAInfo); 272 273 // Build a factor node to remember that this load is independent of the 274 // other one. 275 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 276 Hi.getValue(1)); 277 278 // Handle endianness of the load. 279 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout())) 280 std::swap(Lo, Hi); 281 282 // Modified the chain - switch anything that used the old chain to use 283 // the new one. 284 ReplaceValueWith(SDValue(N, 1), Chain); 285 } 286 287 void DAGTypeLegalizer::ExpandRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) { 288 EVT OVT = N->getValueType(0); 289 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 290 SDValue Chain = N->getOperand(0); 291 SDValue Ptr = N->getOperand(1); 292 SDLoc dl(N); 293 const unsigned Align = N->getConstantOperandVal(3); 294 295 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2), Align); 296 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, N->getOperand(2), 0); 297 Chain = Hi.getValue(1); 298 299 // Handle endianness of the load. 300 if (TLI.hasBigEndianPartOrdering(OVT, DAG.getDataLayout())) 301 std::swap(Lo, Hi); 302 303 // Modified the chain - switch anything that used the old chain to use 304 // the new one. 305 ReplaceValueWith(SDValue(N, 1), Chain); 306 } 307 308 309 //===--------------------------------------------------------------------===// 310 // Generic Operand Expansion. 311 //===--------------------------------------------------------------------===// 312 313 void DAGTypeLegalizer::IntegerToVector(SDValue Op, unsigned NumElements, 314 SmallVectorImpl<SDValue> &Ops, 315 EVT EltVT) { 316 assert(Op.getValueType().isInteger()); 317 SDLoc DL(Op); 318 SDValue Parts[2]; 319 320 if (NumElements > 1) { 321 NumElements >>= 1; 322 SplitInteger(Op, Parts[0], Parts[1]); 323 if (DAG.getDataLayout().isBigEndian()) 324 std::swap(Parts[0], Parts[1]); 325 IntegerToVector(Parts[0], NumElements, Ops, EltVT); 326 IntegerToVector(Parts[1], NumElements, Ops, EltVT); 327 } else { 328 Ops.push_back(DAG.getNode(ISD::BITCAST, DL, EltVT, Op)); 329 } 330 } 331 332 SDValue DAGTypeLegalizer::ExpandOp_BITCAST(SDNode *N) { 333 SDLoc dl(N); 334 if (N->getValueType(0).isVector() && 335 N->getOperand(0).getValueType().isInteger()) { 336 // An illegal expanding type is being converted to a legal vector type. 337 // Make a two element vector out of the expanded parts and convert that 338 // instead, but only if the new vector type is legal (otherwise there 339 // is no point, and it might create expansion loops). For example, on 340 // x86 this turns v1i64 = BITCAST i64 into v1i64 = BITCAST v2i32. 341 // 342 // FIXME: I'm not sure why we are first trying to split the input into 343 // a 2 element vector, so I'm leaving it here to maintain the current 344 // behavior. 345 unsigned NumElts = 2; 346 EVT OVT = N->getOperand(0).getValueType(); 347 EVT NVT = EVT::getVectorVT(*DAG.getContext(), 348 TLI.getTypeToTransformTo(*DAG.getContext(), OVT), 349 NumElts); 350 if (!isTypeLegal(NVT)) { 351 // If we can't find a legal type by splitting the integer in half, 352 // then we can use the node's value type. 353 NumElts = N->getValueType(0).getVectorNumElements(); 354 NVT = N->getValueType(0); 355 } 356 357 SmallVector<SDValue, 8> Ops; 358 IntegerToVector(N->getOperand(0), NumElts, Ops, NVT.getVectorElementType()); 359 360 SDValue Vec = DAG.getBuildVector(NVT, dl, ArrayRef(Ops.data(), NumElts)); 361 return DAG.getNode(ISD::BITCAST, dl, N->getValueType(0), Vec); 362 } 363 364 // Otherwise, store to a temporary and load out again as the new type. 365 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); 366 } 367 368 SDValue DAGTypeLegalizer::ExpandOp_BUILD_VECTOR(SDNode *N) { 369 // The vector type is legal but the element type needs expansion. 370 EVT VecVT = N->getValueType(0); 371 unsigned NumElts = VecVT.getVectorNumElements(); 372 EVT OldVT = N->getOperand(0).getValueType(); 373 EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT); 374 SDLoc dl(N); 375 376 assert(OldVT == VecVT.getVectorElementType() && 377 "BUILD_VECTOR operand type doesn't match vector element type!"); 378 379 // Build a vector of twice the length out of the expanded elements. 380 // For example <3 x i64> -> <6 x i32>. 381 SmallVector<SDValue, 16> NewElts; 382 NewElts.reserve(NumElts*2); 383 384 for (unsigned i = 0; i < NumElts; ++i) { 385 SDValue Lo, Hi; 386 GetExpandedOp(N->getOperand(i), Lo, Hi); 387 if (DAG.getDataLayout().isBigEndian()) 388 std::swap(Lo, Hi); 389 NewElts.push_back(Lo); 390 NewElts.push_back(Hi); 391 } 392 393 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NewElts.size()); 394 SDValue NewVec = DAG.getBuildVector(NewVecVT, dl, NewElts); 395 396 // Convert the new vector to the old vector type. 397 return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec); 398 } 399 400 SDValue DAGTypeLegalizer::ExpandOp_EXTRACT_ELEMENT(SDNode *N) { 401 SDValue Lo, Hi; 402 GetExpandedOp(N->getOperand(0), Lo, Hi); 403 return N->getConstantOperandVal(1) ? Hi : Lo; 404 } 405 406 SDValue DAGTypeLegalizer::ExpandOp_INSERT_VECTOR_ELT(SDNode *N) { 407 // The vector type is legal but the element type needs expansion. 408 EVT VecVT = N->getValueType(0); 409 unsigned NumElts = VecVT.getVectorNumElements(); 410 SDLoc dl(N); 411 412 SDValue Val = N->getOperand(1); 413 EVT OldEVT = Val.getValueType(); 414 EVT NewEVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldEVT); 415 416 assert(OldEVT == VecVT.getVectorElementType() && 417 "Inserted element type doesn't match vector element type!"); 418 419 // Bitconvert to a vector of twice the length with elements of the expanded 420 // type, insert the expanded vector elements, and then convert back. 421 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEVT, NumElts*2); 422 SDValue NewVec = DAG.getNode(ISD::BITCAST, dl, 423 NewVecVT, N->getOperand(0)); 424 425 SDValue Lo, Hi; 426 GetExpandedOp(Val, Lo, Hi); 427 if (DAG.getDataLayout().isBigEndian()) 428 std::swap(Lo, Hi); 429 430 SDValue Idx = N->getOperand(2); 431 Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx); 432 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Lo, Idx); 433 Idx = DAG.getNode(ISD::ADD, dl, 434 Idx.getValueType(), Idx, 435 DAG.getConstant(1, dl, Idx.getValueType())); 436 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Hi, Idx); 437 438 // Convert the new vector to the old vector type. 439 return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec); 440 } 441 442 SDValue DAGTypeLegalizer::ExpandOp_SCALAR_TO_VECTOR(SDNode *N) { 443 SDLoc dl(N); 444 EVT VT = N->getValueType(0); 445 assert(VT.getVectorElementType() == N->getOperand(0).getValueType() && 446 "SCALAR_TO_VECTOR operand type doesn't match vector element type!"); 447 unsigned NumElts = VT.getVectorNumElements(); 448 SmallVector<SDValue, 16> Ops(NumElts); 449 Ops[0] = N->getOperand(0); 450 SDValue UndefVal = DAG.getUNDEF(Ops[0].getValueType()); 451 for (unsigned i = 1; i < NumElts; ++i) 452 Ops[i] = UndefVal; 453 return DAG.getBuildVector(VT, dl, Ops); 454 } 455 456 SDValue DAGTypeLegalizer::ExpandOp_NormalStore(SDNode *N, unsigned OpNo) { 457 assert(ISD::isNormalStore(N) && "This routine only for normal stores!"); 458 assert(OpNo == 1 && "Can only expand the stored value so far"); 459 SDLoc dl(N); 460 461 StoreSDNode *St = cast<StoreSDNode>(N); 462 assert(!St->isAtomic() && "Atomics can not be split"); 463 EVT ValueVT = St->getValue().getValueType(); 464 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT); 465 SDValue Chain = St->getChain(); 466 SDValue Ptr = St->getBasePtr(); 467 AAMDNodes AAInfo = St->getAAInfo(); 468 469 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 470 unsigned IncrementSize = NVT.getSizeInBits() / 8; 471 472 SDValue Lo, Hi; 473 GetExpandedOp(St->getValue(), Lo, Hi); 474 475 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout())) 476 std::swap(Lo, Hi); 477 478 Lo = DAG.getStore(Chain, dl, Lo, Ptr, St->getPointerInfo(), 479 St->getOriginalAlign(), St->getMemOperand()->getFlags(), 480 AAInfo); 481 482 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize)); 483 Hi = DAG.getStore( 484 Chain, dl, Hi, Ptr, St->getPointerInfo().getWithOffset(IncrementSize), 485 St->getOriginalAlign(), St->getMemOperand()->getFlags(), AAInfo); 486 487 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 488 } 489 490 491 //===--------------------------------------------------------------------===// 492 // Generic Result Splitting. 493 //===--------------------------------------------------------------------===// 494 495 // Be careful to make no assumptions about which of Lo/Hi is stored first in 496 // memory (for vectors it is always Lo first followed by Hi in the following 497 // bytes; for integers and floats it is Lo first if and only if the machine is 498 // little-endian). 499 500 void DAGTypeLegalizer::SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo, 501 SDValue &Lo, SDValue &Hi) { 502 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 503 GetSplitOp(Op, Lo, Hi); 504 } 505 506 void DAGTypeLegalizer::SplitRes_Select(SDNode *N, SDValue &Lo, SDValue &Hi) { 507 SDValue LL, LH, RL, RH, CL, CH; 508 SDLoc dl(N); 509 unsigned Opcode = N->getOpcode(); 510 GetSplitOp(N->getOperand(1), LL, LH); 511 GetSplitOp(N->getOperand(2), RL, RH); 512 513 SDValue Cond = N->getOperand(0); 514 CL = CH = Cond; 515 if (Cond.getValueType().isVector()) { 516 if (SDValue Res = WidenVSELECTMask(N)) 517 std::tie(CL, CH) = DAG.SplitVector(Res, dl); 518 // Check if there are already splitted versions of the vector available and 519 // use those instead of splitting the mask operand again. 520 else if (getTypeAction(Cond.getValueType()) == 521 TargetLowering::TypeSplitVector) 522 GetSplitVector(Cond, CL, CH); 523 // It seems to improve code to generate two narrow SETCCs as opposed to 524 // splitting a wide result vector. 525 else if (Cond.getOpcode() == ISD::SETCC) { 526 // If the condition is a vXi1 vector, and the LHS of the setcc is a legal 527 // type and the setcc result type is the same vXi1, then leave the setcc 528 // alone. 529 EVT CondLHSVT = Cond.getOperand(0).getValueType(); 530 if (Cond.getValueType().getVectorElementType() == MVT::i1 && 531 isTypeLegal(CondLHSVT) && 532 getSetCCResultType(CondLHSVT) == Cond.getValueType()) 533 std::tie(CL, CH) = DAG.SplitVector(Cond, dl); 534 else 535 SplitVecRes_SETCC(Cond.getNode(), CL, CH); 536 } else 537 std::tie(CL, CH) = DAG.SplitVector(Cond, dl); 538 } 539 540 if (Opcode != ISD::VP_SELECT && Opcode != ISD::VP_MERGE) { 541 Lo = DAG.getNode(Opcode, dl, LL.getValueType(), CL, LL, RL); 542 Hi = DAG.getNode(Opcode, dl, LH.getValueType(), CH, LH, RH); 543 return; 544 } 545 546 SDValue EVLLo, EVLHi; 547 std::tie(EVLLo, EVLHi) = 548 DAG.SplitEVL(N->getOperand(3), N->getValueType(0), dl); 549 550 Lo = DAG.getNode(Opcode, dl, LL.getValueType(), CL, LL, RL, EVLLo); 551 Hi = DAG.getNode(Opcode, dl, LH.getValueType(), CH, LH, RH, EVLHi); 552 } 553 554 void DAGTypeLegalizer::SplitRes_SELECT_CC(SDNode *N, SDValue &Lo, 555 SDValue &Hi) { 556 SDValue LL, LH, RL, RH; 557 SDLoc dl(N); 558 GetSplitOp(N->getOperand(2), LL, LH); 559 GetSplitOp(N->getOperand(3), RL, RH); 560 561 Lo = DAG.getNode(ISD::SELECT_CC, dl, LL.getValueType(), N->getOperand(0), 562 N->getOperand(1), LL, RL, N->getOperand(4)); 563 Hi = DAG.getNode(ISD::SELECT_CC, dl, LH.getValueType(), N->getOperand(0), 564 N->getOperand(1), LH, RH, N->getOperand(4)); 565 } 566 567 void DAGTypeLegalizer::SplitRes_UNDEF(SDNode *N, SDValue &Lo, SDValue &Hi) { 568 EVT LoVT, HiVT; 569 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 570 Lo = DAG.getUNDEF(LoVT); 571 Hi = DAG.getUNDEF(HiVT); 572 } 573 574 void DAGTypeLegalizer::SplitVecRes_AssertZext(SDNode *N, SDValue &Lo, 575 SDValue &Hi) { 576 SDValue L, H; 577 SDLoc dl(N); 578 GetSplitOp(N->getOperand(0), L, H); 579 580 Lo = DAG.getNode(ISD::AssertZext, dl, L.getValueType(), L, N->getOperand(1)); 581 Hi = DAG.getNode(ISD::AssertZext, dl, H.getValueType(), H, N->getOperand(1)); 582 } 583 584 void DAGTypeLegalizer::SplitRes_FREEZE(SDNode *N, SDValue &Lo, SDValue &Hi) { 585 SDValue L, H; 586 SDLoc dl(N); 587 GetSplitOp(N->getOperand(0), L, H); 588 589 Lo = DAG.getNode(ISD::FREEZE, dl, L.getValueType(), L); 590 Hi = DAG.getNode(ISD::FREEZE, dl, H.getValueType(), H); 591 } 592 593 void DAGTypeLegalizer::SplitRes_ARITH_FENCE(SDNode *N, SDValue &Lo, 594 SDValue &Hi) { 595 SDValue L, H; 596 SDLoc DL(N); 597 GetSplitOp(N->getOperand(0), L, H); 598 599 Lo = DAG.getNode(ISD::ARITH_FENCE, DL, L.getValueType(), L); 600 Hi = DAG.getNode(ISD::ARITH_FENCE, DL, H.getValueType(), H); 601 } 602