xref: /freebsd/contrib/llvm-project/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp (revision 0d8fe2373503aeac48492f28073049a8bfa4feb5)
1 //===-- HexagonISelLowering.cpp - Hexagon DAG Lowering Implementation -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the interfaces that Hexagon uses to lower LLVM code
10 // into a selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "HexagonISelLowering.h"
15 #include "Hexagon.h"
16 #include "HexagonMachineFunctionInfo.h"
17 #include "HexagonRegisterInfo.h"
18 #include "HexagonSubtarget.h"
19 #include "HexagonTargetMachine.h"
20 #include "HexagonTargetObjectFile.h"
21 #include "llvm/ADT/APInt.h"
22 #include "llvm/ADT/ArrayRef.h"
23 #include "llvm/ADT/SmallVector.h"
24 #include "llvm/ADT/StringSwitch.h"
25 #include "llvm/CodeGen/CallingConvLower.h"
26 #include "llvm/CodeGen/MachineFrameInfo.h"
27 #include "llvm/CodeGen/MachineFunction.h"
28 #include "llvm/CodeGen/MachineMemOperand.h"
29 #include "llvm/CodeGen/MachineRegisterInfo.h"
30 #include "llvm/CodeGen/RuntimeLibcalls.h"
31 #include "llvm/CodeGen/SelectionDAG.h"
32 #include "llvm/CodeGen/TargetCallingConv.h"
33 #include "llvm/CodeGen/ValueTypes.h"
34 #include "llvm/IR/BasicBlock.h"
35 #include "llvm/IR/CallingConv.h"
36 #include "llvm/IR/DataLayout.h"
37 #include "llvm/IR/DerivedTypes.h"
38 #include "llvm/IR/Function.h"
39 #include "llvm/IR/GlobalValue.h"
40 #include "llvm/IR/InlineAsm.h"
41 #include "llvm/IR/Instructions.h"
42 #include "llvm/IR/IntrinsicInst.h"
43 #include "llvm/IR/Intrinsics.h"
44 #include "llvm/IR/IntrinsicsHexagon.h"
45 #include "llvm/IR/Module.h"
46 #include "llvm/IR/Type.h"
47 #include "llvm/IR/Value.h"
48 #include "llvm/MC/MCRegisterInfo.h"
49 #include "llvm/Support/Casting.h"
50 #include "llvm/Support/CodeGen.h"
51 #include "llvm/Support/CommandLine.h"
52 #include "llvm/Support/Debug.h"
53 #include "llvm/Support/ErrorHandling.h"
54 #include "llvm/Support/MathExtras.h"
55 #include "llvm/Support/raw_ostream.h"
56 #include "llvm/Target/TargetMachine.h"
57 #include <algorithm>
58 #include <cassert>
59 #include <cstddef>
60 #include <cstdint>
61 #include <limits>
62 #include <utility>
63 
64 using namespace llvm;
65 
66 #define DEBUG_TYPE "hexagon-lowering"
67 
68 static cl::opt<bool> EmitJumpTables("hexagon-emit-jump-tables",
69   cl::init(true), cl::Hidden,
70   cl::desc("Control jump table emission on Hexagon target"));
71 
72 static cl::opt<bool> EnableHexSDNodeSched("enable-hexagon-sdnode-sched",
73   cl::Hidden, cl::ZeroOrMore, cl::init(false),
74   cl::desc("Enable Hexagon SDNode scheduling"));
75 
76 static cl::opt<bool> EnableFastMath("ffast-math",
77   cl::Hidden, cl::ZeroOrMore, cl::init(false),
78   cl::desc("Enable Fast Math processing"));
79 
80 static cl::opt<int> MinimumJumpTables("minimum-jump-tables",
81   cl::Hidden, cl::ZeroOrMore, cl::init(5),
82   cl::desc("Set minimum jump tables"));
83 
84 static cl::opt<int> MaxStoresPerMemcpyCL("max-store-memcpy",
85   cl::Hidden, cl::ZeroOrMore, cl::init(6),
86   cl::desc("Max #stores to inline memcpy"));
87 
88 static cl::opt<int> MaxStoresPerMemcpyOptSizeCL("max-store-memcpy-Os",
89   cl::Hidden, cl::ZeroOrMore, cl::init(4),
90   cl::desc("Max #stores to inline memcpy"));
91 
92 static cl::opt<int> MaxStoresPerMemmoveCL("max-store-memmove",
93   cl::Hidden, cl::ZeroOrMore, cl::init(6),
94   cl::desc("Max #stores to inline memmove"));
95 
96 static cl::opt<int> MaxStoresPerMemmoveOptSizeCL("max-store-memmove-Os",
97   cl::Hidden, cl::ZeroOrMore, cl::init(4),
98   cl::desc("Max #stores to inline memmove"));
99 
100 static cl::opt<int> MaxStoresPerMemsetCL("max-store-memset",
101   cl::Hidden, cl::ZeroOrMore, cl::init(8),
102   cl::desc("Max #stores to inline memset"));
103 
104 static cl::opt<int> MaxStoresPerMemsetOptSizeCL("max-store-memset-Os",
105   cl::Hidden, cl::ZeroOrMore, cl::init(4),
106   cl::desc("Max #stores to inline memset"));
107 
108 static cl::opt<bool> AlignLoads("hexagon-align-loads",
109   cl::Hidden, cl::init(false),
110   cl::desc("Rewrite unaligned loads as a pair of aligned loads"));
111 
112 static cl::opt<bool>
113     DisableArgsMinAlignment("hexagon-disable-args-min-alignment", cl::Hidden,
114                             cl::init(false),
115                             cl::desc("Disable minimum alignment of 1 for "
116                                      "arguments passed by value on stack"));
117 
118 namespace {
119 
120   class HexagonCCState : public CCState {
121     unsigned NumNamedVarArgParams = 0;
122 
123   public:
124     HexagonCCState(CallingConv::ID CC, bool IsVarArg, MachineFunction &MF,
125                    SmallVectorImpl<CCValAssign> &locs, LLVMContext &C,
126                    unsigned NumNamedArgs)
127         : CCState(CC, IsVarArg, MF, locs, C),
128           NumNamedVarArgParams(NumNamedArgs) {}
129     unsigned getNumNamedVarArgParams() const { return NumNamedVarArgParams; }
130   };
131 
132 } // end anonymous namespace
133 
134 
135 // Implement calling convention for Hexagon.
136 
137 static bool CC_SkipOdd(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
138                        CCValAssign::LocInfo &LocInfo,
139                        ISD::ArgFlagsTy &ArgFlags, CCState &State) {
140   static const MCPhysReg ArgRegs[] = {
141     Hexagon::R0, Hexagon::R1, Hexagon::R2,
142     Hexagon::R3, Hexagon::R4, Hexagon::R5
143   };
144   const unsigned NumArgRegs = array_lengthof(ArgRegs);
145   unsigned RegNum = State.getFirstUnallocated(ArgRegs);
146 
147   // RegNum is an index into ArgRegs: skip a register if RegNum is odd.
148   if (RegNum != NumArgRegs && RegNum % 2 == 1)
149     State.AllocateReg(ArgRegs[RegNum]);
150 
151   // Always return false here, as this function only makes sure that the first
152   // unallocated register has an even register number and does not actually
153   // allocate a register for the current argument.
154   return false;
155 }
156 
157 #include "HexagonGenCallingConv.inc"
158 
159 
160 SDValue
161 HexagonTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG)
162       const {
163   return SDValue();
164 }
165 
166 /// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified
167 /// by "Src" to address "Dst" of size "Size".  Alignment information is
168 /// specified by the specific parameter attribute. The copy will be passed as
169 /// a byval function parameter.  Sometimes what we are copying is the end of a
170 /// larger object, the part that does not fit in registers.
171 static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
172                                          SDValue Chain, ISD::ArgFlagsTy Flags,
173                                          SelectionDAG &DAG, const SDLoc &dl) {
174   SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
175   return DAG.getMemcpy(
176       Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(),
177       /*isVolatile=*/false, /*AlwaysInline=*/false,
178       /*isTailCall=*/false, MachinePointerInfo(), MachinePointerInfo());
179 }
180 
181 bool
182 HexagonTargetLowering::CanLowerReturn(
183     CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
184     const SmallVectorImpl<ISD::OutputArg> &Outs,
185     LLVMContext &Context) const {
186   SmallVector<CCValAssign, 16> RVLocs;
187   CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
188 
189   if (MF.getSubtarget<HexagonSubtarget>().useHVXOps())
190     return CCInfo.CheckReturn(Outs, RetCC_Hexagon_HVX);
191   return CCInfo.CheckReturn(Outs, RetCC_Hexagon);
192 }
193 
194 // LowerReturn - Lower ISD::RET. If a struct is larger than 8 bytes and is
195 // passed by value, the function prototype is modified to return void and
196 // the value is stored in memory pointed by a pointer passed by caller.
197 SDValue
198 HexagonTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
199                                    bool IsVarArg,
200                                    const SmallVectorImpl<ISD::OutputArg> &Outs,
201                                    const SmallVectorImpl<SDValue> &OutVals,
202                                    const SDLoc &dl, SelectionDAG &DAG) const {
203   // CCValAssign - represent the assignment of the return value to locations.
204   SmallVector<CCValAssign, 16> RVLocs;
205 
206   // CCState - Info about the registers and stack slot.
207   CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
208                  *DAG.getContext());
209 
210   // Analyze return values of ISD::RET
211   if (Subtarget.useHVXOps())
212     CCInfo.AnalyzeReturn(Outs, RetCC_Hexagon_HVX);
213   else
214     CCInfo.AnalyzeReturn(Outs, RetCC_Hexagon);
215 
216   SDValue Flag;
217   SmallVector<SDValue, 4> RetOps(1, Chain);
218 
219   // Copy the result values into the output registers.
220   for (unsigned i = 0; i != RVLocs.size(); ++i) {
221     CCValAssign &VA = RVLocs[i];
222 
223     Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), OutVals[i], Flag);
224 
225     // Guarantee that all emitted copies are stuck together with flags.
226     Flag = Chain.getValue(1);
227     RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
228   }
229 
230   RetOps[0] = Chain;  // Update chain.
231 
232   // Add the flag if we have it.
233   if (Flag.getNode())
234     RetOps.push_back(Flag);
235 
236   return DAG.getNode(HexagonISD::RET_FLAG, dl, MVT::Other, RetOps);
237 }
238 
239 bool HexagonTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
240   // If either no tail call or told not to tail call at all, don't.
241   return CI->isTailCall();
242 }
243 
244 Register HexagonTargetLowering::getRegisterByName(
245       const char* RegName, LLT VT, const MachineFunction &) const {
246   // Just support r19, the linux kernel uses it.
247   Register Reg = StringSwitch<Register>(RegName)
248                      .Case("r0", Hexagon::R0)
249                      .Case("r1", Hexagon::R1)
250                      .Case("r2", Hexagon::R2)
251                      .Case("r3", Hexagon::R3)
252                      .Case("r4", Hexagon::R4)
253                      .Case("r5", Hexagon::R5)
254                      .Case("r6", Hexagon::R6)
255                      .Case("r7", Hexagon::R7)
256                      .Case("r8", Hexagon::R8)
257                      .Case("r9", Hexagon::R9)
258                      .Case("r10", Hexagon::R10)
259                      .Case("r11", Hexagon::R11)
260                      .Case("r12", Hexagon::R12)
261                      .Case("r13", Hexagon::R13)
262                      .Case("r14", Hexagon::R14)
263                      .Case("r15", Hexagon::R15)
264                      .Case("r16", Hexagon::R16)
265                      .Case("r17", Hexagon::R17)
266                      .Case("r18", Hexagon::R18)
267                      .Case("r19", Hexagon::R19)
268                      .Case("r20", Hexagon::R20)
269                      .Case("r21", Hexagon::R21)
270                      .Case("r22", Hexagon::R22)
271                      .Case("r23", Hexagon::R23)
272                      .Case("r24", Hexagon::R24)
273                      .Case("r25", Hexagon::R25)
274                      .Case("r26", Hexagon::R26)
275                      .Case("r27", Hexagon::R27)
276                      .Case("r28", Hexagon::R28)
277                      .Case("r29", Hexagon::R29)
278                      .Case("r30", Hexagon::R30)
279                      .Case("r31", Hexagon::R31)
280                      .Case("r1:0", Hexagon::D0)
281                      .Case("r3:2", Hexagon::D1)
282                      .Case("r5:4", Hexagon::D2)
283                      .Case("r7:6", Hexagon::D3)
284                      .Case("r9:8", Hexagon::D4)
285                      .Case("r11:10", Hexagon::D5)
286                      .Case("r13:12", Hexagon::D6)
287                      .Case("r15:14", Hexagon::D7)
288                      .Case("r17:16", Hexagon::D8)
289                      .Case("r19:18", Hexagon::D9)
290                      .Case("r21:20", Hexagon::D10)
291                      .Case("r23:22", Hexagon::D11)
292                      .Case("r25:24", Hexagon::D12)
293                      .Case("r27:26", Hexagon::D13)
294                      .Case("r29:28", Hexagon::D14)
295                      .Case("r31:30", Hexagon::D15)
296                      .Case("sp", Hexagon::R29)
297                      .Case("fp", Hexagon::R30)
298                      .Case("lr", Hexagon::R31)
299                      .Case("p0", Hexagon::P0)
300                      .Case("p1", Hexagon::P1)
301                      .Case("p2", Hexagon::P2)
302                      .Case("p3", Hexagon::P3)
303                      .Case("sa0", Hexagon::SA0)
304                      .Case("lc0", Hexagon::LC0)
305                      .Case("sa1", Hexagon::SA1)
306                      .Case("lc1", Hexagon::LC1)
307                      .Case("m0", Hexagon::M0)
308                      .Case("m1", Hexagon::M1)
309                      .Case("usr", Hexagon::USR)
310                      .Case("ugp", Hexagon::UGP)
311                      .Default(Register());
312   if (Reg)
313     return Reg;
314 
315   report_fatal_error("Invalid register name global variable");
316 }
317 
318 /// LowerCallResult - Lower the result values of an ISD::CALL into the
319 /// appropriate copies out of appropriate physical registers.  This assumes that
320 /// Chain/Glue are the input chain/glue to use, and that TheCall is the call
321 /// being lowered. Returns a SDNode with the same number of values as the
322 /// ISD::CALL.
323 SDValue HexagonTargetLowering::LowerCallResult(
324     SDValue Chain, SDValue Glue, CallingConv::ID CallConv, bool IsVarArg,
325     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
326     SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
327     const SmallVectorImpl<SDValue> &OutVals, SDValue Callee) const {
328   // Assign locations to each value returned by this call.
329   SmallVector<CCValAssign, 16> RVLocs;
330 
331   CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
332                  *DAG.getContext());
333 
334   if (Subtarget.useHVXOps())
335     CCInfo.AnalyzeCallResult(Ins, RetCC_Hexagon_HVX);
336   else
337     CCInfo.AnalyzeCallResult(Ins, RetCC_Hexagon);
338 
339   // Copy all of the result registers out of their specified physreg.
340   for (unsigned i = 0; i != RVLocs.size(); ++i) {
341     SDValue RetVal;
342     if (RVLocs[i].getValVT() == MVT::i1) {
343       // Return values of type MVT::i1 require special handling. The reason
344       // is that MVT::i1 is associated with the PredRegs register class, but
345       // values of that type are still returned in R0. Generate an explicit
346       // copy into a predicate register from R0, and treat the value of the
347       // predicate register as the call result.
348       auto &MRI = DAG.getMachineFunction().getRegInfo();
349       SDValue FR0 = DAG.getCopyFromReg(Chain, dl, RVLocs[i].getLocReg(),
350                                        MVT::i32, Glue);
351       // FR0 = (Value, Chain, Glue)
352       Register PredR = MRI.createVirtualRegister(&Hexagon::PredRegsRegClass);
353       SDValue TPR = DAG.getCopyToReg(FR0.getValue(1), dl, PredR,
354                                      FR0.getValue(0), FR0.getValue(2));
355       // TPR = (Chain, Glue)
356       // Don't glue this CopyFromReg, because it copies from a virtual
357       // register. If it is glued to the call, InstrEmitter will add it
358       // as an implicit def to the call (EmitMachineNode).
359       RetVal = DAG.getCopyFromReg(TPR.getValue(0), dl, PredR, MVT::i1);
360       Glue = TPR.getValue(1);
361       Chain = TPR.getValue(0);
362     } else {
363       RetVal = DAG.getCopyFromReg(Chain, dl, RVLocs[i].getLocReg(),
364                                   RVLocs[i].getValVT(), Glue);
365       Glue = RetVal.getValue(2);
366       Chain = RetVal.getValue(1);
367     }
368     InVals.push_back(RetVal.getValue(0));
369   }
370 
371   return Chain;
372 }
373 
374 /// LowerCall - Functions arguments are copied from virtual regs to
375 /// (physical regs)/(stack frame), CALLSEQ_START and CALLSEQ_END are emitted.
376 SDValue
377 HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
378                                  SmallVectorImpl<SDValue> &InVals) const {
379   SelectionDAG &DAG                     = CLI.DAG;
380   SDLoc &dl                             = CLI.DL;
381   SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
382   SmallVectorImpl<SDValue> &OutVals     = CLI.OutVals;
383   SmallVectorImpl<ISD::InputArg> &Ins   = CLI.Ins;
384   SDValue Chain                         = CLI.Chain;
385   SDValue Callee                        = CLI.Callee;
386   CallingConv::ID CallConv              = CLI.CallConv;
387   bool IsVarArg                         = CLI.IsVarArg;
388   bool DoesNotReturn                    = CLI.DoesNotReturn;
389 
390   bool IsStructRet    = Outs.empty() ? false : Outs[0].Flags.isSRet();
391   MachineFunction &MF = DAG.getMachineFunction();
392   MachineFrameInfo &MFI = MF.getFrameInfo();
393   auto PtrVT = getPointerTy(MF.getDataLayout());
394 
395   unsigned NumParams = CLI.CB ? CLI.CB->getFunctionType()->getNumParams() : 0;
396   if (GlobalAddressSDNode *GAN = dyn_cast<GlobalAddressSDNode>(Callee))
397     Callee = DAG.getTargetGlobalAddress(GAN->getGlobal(), dl, MVT::i32);
398 
399   // Linux ABI treats var-arg calls the same way as regular ones.
400   bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;
401 
402   // Analyze operands of the call, assigning locations to each operand.
403   SmallVector<CCValAssign, 16> ArgLocs;
404   HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext(),
405                         NumParams);
406 
407   if (Subtarget.useHVXOps())
408     CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_HVX);
409   else if (DisableArgsMinAlignment)
410     CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_Legacy);
411   else
412     CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon);
413 
414   if (CLI.IsTailCall) {
415     bool StructAttrFlag = MF.getFunction().hasStructRetAttr();
416     CLI.IsTailCall = IsEligibleForTailCallOptimization(Callee, CallConv,
417                         IsVarArg, IsStructRet, StructAttrFlag, Outs,
418                         OutVals, Ins, DAG);
419     for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
420       CCValAssign &VA = ArgLocs[i];
421       if (VA.isMemLoc()) {
422         CLI.IsTailCall = false;
423         break;
424       }
425     }
426     LLVM_DEBUG(dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n"
427                                          : "Argument must be passed on stack. "
428                                            "Not eligible for Tail Call\n"));
429   }
430   // Get a count of how many bytes are to be pushed on the stack.
431   unsigned NumBytes = CCInfo.getNextStackOffset();
432   SmallVector<std::pair<unsigned, SDValue>, 16> RegsToPass;
433   SmallVector<SDValue, 8> MemOpChains;
434 
435   const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
436   SDValue StackPtr =
437       DAG.getCopyFromReg(Chain, dl, HRI.getStackRegister(), PtrVT);
438 
439   bool NeedsArgAlign = false;
440   Align LargestAlignSeen;
441   // Walk the register/memloc assignments, inserting copies/loads.
442   for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
443     CCValAssign &VA = ArgLocs[i];
444     SDValue Arg = OutVals[i];
445     ISD::ArgFlagsTy Flags = Outs[i].Flags;
446     // Record if we need > 8 byte alignment on an argument.
447     bool ArgAlign = Subtarget.isHVXVectorType(VA.getValVT());
448     NeedsArgAlign |= ArgAlign;
449 
450     // Promote the value if needed.
451     switch (VA.getLocInfo()) {
452       default:
453         // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt.
454         llvm_unreachable("Unknown loc info!");
455       case CCValAssign::Full:
456         break;
457       case CCValAssign::BCvt:
458         Arg = DAG.getBitcast(VA.getLocVT(), Arg);
459         break;
460       case CCValAssign::SExt:
461         Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg);
462         break;
463       case CCValAssign::ZExt:
464         Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg);
465         break;
466       case CCValAssign::AExt:
467         Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg);
468         break;
469     }
470 
471     if (VA.isMemLoc()) {
472       unsigned LocMemOffset = VA.getLocMemOffset();
473       SDValue MemAddr = DAG.getConstant(LocMemOffset, dl,
474                                         StackPtr.getValueType());
475       MemAddr = DAG.getNode(ISD::ADD, dl, MVT::i32, StackPtr, MemAddr);
476       if (ArgAlign)
477         LargestAlignSeen = std::max(
478             LargestAlignSeen, Align(VA.getLocVT().getStoreSizeInBits() / 8));
479       if (Flags.isByVal()) {
480         // The argument is a struct passed by value. According to LLVM, "Arg"
481         // is a pointer.
482         MemOpChains.push_back(CreateCopyOfByValArgument(Arg, MemAddr, Chain,
483                                                         Flags, DAG, dl));
484       } else {
485         MachinePointerInfo LocPI = MachinePointerInfo::getStack(
486             DAG.getMachineFunction(), LocMemOffset);
487         SDValue S = DAG.getStore(Chain, dl, Arg, MemAddr, LocPI);
488         MemOpChains.push_back(S);
489       }
490       continue;
491     }
492 
493     // Arguments that can be passed on register must be kept at RegsToPass
494     // vector.
495     if (VA.isRegLoc())
496       RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
497   }
498 
499   if (NeedsArgAlign && Subtarget.hasV60Ops()) {
500     LLVM_DEBUG(dbgs() << "Function needs byte stack align due to call args\n");
501     Align VecAlign(HRI.getSpillAlignment(Hexagon::HvxVRRegClass));
502     LargestAlignSeen = std::max(LargestAlignSeen, VecAlign);
503     MFI.ensureMaxAlignment(LargestAlignSeen);
504   }
505   // Transform all store nodes into one single node because all store
506   // nodes are independent of each other.
507   if (!MemOpChains.empty())
508     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
509 
510   SDValue Glue;
511   if (!CLI.IsTailCall) {
512     Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl);
513     Glue = Chain.getValue(1);
514   }
515 
516   // Build a sequence of copy-to-reg nodes chained together with token
517   // chain and flag operands which copy the outgoing args into registers.
518   // The Glue is necessary since all emitted instructions must be
519   // stuck together.
520   if (!CLI.IsTailCall) {
521     for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
522       Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
523                                RegsToPass[i].second, Glue);
524       Glue = Chain.getValue(1);
525     }
526   } else {
527     // For tail calls lower the arguments to the 'real' stack slot.
528     //
529     // Force all the incoming stack arguments to be loaded from the stack
530     // before any new outgoing arguments are stored to the stack, because the
531     // outgoing stack slots may alias the incoming argument stack slots, and
532     // the alias isn't otherwise explicit. This is slightly more conservative
533     // than necessary, because it means that each store effectively depends
534     // on every argument instead of just those arguments it would clobber.
535     //
536     // Do not flag preceding copytoreg stuff together with the following stuff.
537     Glue = SDValue();
538     for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
539       Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
540                                RegsToPass[i].second, Glue);
541       Glue = Chain.getValue(1);
542     }
543     Glue = SDValue();
544   }
545 
546   bool LongCalls = MF.getSubtarget<HexagonSubtarget>().useLongCalls();
547   unsigned Flags = LongCalls ? HexagonII::HMOTF_ConstExtended : 0;
548 
549   // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
550   // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
551   // node so that legalize doesn't hack it.
552   if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
553     Callee = DAG.getTargetGlobalAddress(G->getGlobal(), dl, PtrVT, 0, Flags);
554   } else if (ExternalSymbolSDNode *S =
555              dyn_cast<ExternalSymbolSDNode>(Callee)) {
556     Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, Flags);
557   }
558 
559   // Returns a chain & a flag for retval copy to use.
560   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
561   SmallVector<SDValue, 8> Ops;
562   Ops.push_back(Chain);
563   Ops.push_back(Callee);
564 
565   // Add argument registers to the end of the list so that they are
566   // known live into the call.
567   for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
568     Ops.push_back(DAG.getRegister(RegsToPass[i].first,
569                                   RegsToPass[i].second.getValueType()));
570   }
571 
572   const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallConv);
573   assert(Mask && "Missing call preserved mask for calling convention");
574   Ops.push_back(DAG.getRegisterMask(Mask));
575 
576   if (Glue.getNode())
577     Ops.push_back(Glue);
578 
579   if (CLI.IsTailCall) {
580     MFI.setHasTailCall();
581     return DAG.getNode(HexagonISD::TC_RETURN, dl, NodeTys, Ops);
582   }
583 
584   // Set this here because we need to know this for "hasFP" in frame lowering.
585   // The target-independent code calls getFrameRegister before setting it, and
586   // getFrameRegister uses hasFP to determine whether the function has FP.
587   MFI.setHasCalls(true);
588 
589   unsigned OpCode = DoesNotReturn ? HexagonISD::CALLnr : HexagonISD::CALL;
590   Chain = DAG.getNode(OpCode, dl, NodeTys, Ops);
591   Glue = Chain.getValue(1);
592 
593   // Create the CALLSEQ_END node.
594   Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true),
595                              DAG.getIntPtrConstant(0, dl, true), Glue, dl);
596   Glue = Chain.getValue(1);
597 
598   // Handle result values, copying them out of physregs into vregs that we
599   // return.
600   return LowerCallResult(Chain, Glue, CallConv, IsVarArg, Ins, dl, DAG,
601                          InVals, OutVals, Callee);
602 }
603 
604 /// Returns true by value, base pointer and offset pointer and addressing
605 /// mode by reference if this node can be combined with a load / store to
606 /// form a post-indexed load / store.
607 bool HexagonTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
608       SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM,
609       SelectionDAG &DAG) const {
610   LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(N);
611   if (!LSN)
612     return false;
613   EVT VT = LSN->getMemoryVT();
614   if (!VT.isSimple())
615     return false;
616   bool IsLegalType = VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 ||
617                      VT == MVT::i64 || VT == MVT::f32 || VT == MVT::f64 ||
618                      VT == MVT::v2i16 || VT == MVT::v2i32 || VT == MVT::v4i8 ||
619                      VT == MVT::v4i16 || VT == MVT::v8i8 ||
620                      Subtarget.isHVXVectorType(VT.getSimpleVT());
621   if (!IsLegalType)
622     return false;
623 
624   if (Op->getOpcode() != ISD::ADD)
625     return false;
626   Base = Op->getOperand(0);
627   Offset = Op->getOperand(1);
628   if (!isa<ConstantSDNode>(Offset.getNode()))
629     return false;
630   AM = ISD::POST_INC;
631 
632   int32_t V = cast<ConstantSDNode>(Offset.getNode())->getSExtValue();
633   return Subtarget.getInstrInfo()->isValidAutoIncImm(VT, V);
634 }
635 
636 SDValue
637 HexagonTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
638   MachineFunction &MF = DAG.getMachineFunction();
639   auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();
640   const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
641   unsigned LR = HRI.getRARegister();
642 
643   if ((Op.getOpcode() != ISD::INLINEASM &&
644        Op.getOpcode() != ISD::INLINEASM_BR) || HMFI.hasClobberLR())
645     return Op;
646 
647   unsigned NumOps = Op.getNumOperands();
648   if (Op.getOperand(NumOps-1).getValueType() == MVT::Glue)
649     --NumOps;  // Ignore the flag operand.
650 
651   for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
652     unsigned Flags = cast<ConstantSDNode>(Op.getOperand(i))->getZExtValue();
653     unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
654     ++i;  // Skip the ID value.
655 
656     switch (InlineAsm::getKind(Flags)) {
657       default:
658         llvm_unreachable("Bad flags!");
659       case InlineAsm::Kind_RegUse:
660       case InlineAsm::Kind_Imm:
661       case InlineAsm::Kind_Mem:
662         i += NumVals;
663         break;
664       case InlineAsm::Kind_Clobber:
665       case InlineAsm::Kind_RegDef:
666       case InlineAsm::Kind_RegDefEarlyClobber: {
667         for (; NumVals; --NumVals, ++i) {
668           unsigned Reg = cast<RegisterSDNode>(Op.getOperand(i))->getReg();
669           if (Reg != LR)
670             continue;
671           HMFI.setHasClobberLR(true);
672           return Op;
673         }
674         break;
675       }
676     }
677   }
678 
679   return Op;
680 }
681 
682 // Need to transform ISD::PREFETCH into something that doesn't inherit
683 // all of the properties of ISD::PREFETCH, specifically SDNPMayLoad and
684 // SDNPMayStore.
685 SDValue HexagonTargetLowering::LowerPREFETCH(SDValue Op,
686                                              SelectionDAG &DAG) const {
687   SDValue Chain = Op.getOperand(0);
688   SDValue Addr = Op.getOperand(1);
689   // Lower it to DCFETCH($reg, #0).  A "pat" will try to merge the offset in,
690   // if the "reg" is fed by an "add".
691   SDLoc DL(Op);
692   SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
693   return DAG.getNode(HexagonISD::DCFETCH, DL, MVT::Other, Chain, Addr, Zero);
694 }
695 
696 // Custom-handle ISD::READCYCLECOUNTER because the target-independent SDNode
697 // is marked as having side-effects, while the register read on Hexagon does
698 // not have any. TableGen refuses to accept the direct pattern from that node
699 // to the A4_tfrcpp.
700 SDValue HexagonTargetLowering::LowerREADCYCLECOUNTER(SDValue Op,
701                                                      SelectionDAG &DAG) const {
702   SDValue Chain = Op.getOperand(0);
703   SDLoc dl(Op);
704   SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
705   return DAG.getNode(HexagonISD::READCYCLE, dl, VTs, Chain);
706 }
707 
708 SDValue HexagonTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
709       SelectionDAG &DAG) const {
710   SDValue Chain = Op.getOperand(0);
711   unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
712   // Lower the hexagon_prefetch builtin to DCFETCH, as above.
713   if (IntNo == Intrinsic::hexagon_prefetch) {
714     SDValue Addr = Op.getOperand(2);
715     SDLoc DL(Op);
716     SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
717     return DAG.getNode(HexagonISD::DCFETCH, DL, MVT::Other, Chain, Addr, Zero);
718   }
719   return SDValue();
720 }
721 
722 SDValue
723 HexagonTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
724                                                SelectionDAG &DAG) const {
725   SDValue Chain = Op.getOperand(0);
726   SDValue Size = Op.getOperand(1);
727   SDValue Align = Op.getOperand(2);
728   SDLoc dl(Op);
729 
730   ConstantSDNode *AlignConst = dyn_cast<ConstantSDNode>(Align);
731   assert(AlignConst && "Non-constant Align in LowerDYNAMIC_STACKALLOC");
732 
733   unsigned A = AlignConst->getSExtValue();
734   auto &HFI = *Subtarget.getFrameLowering();
735   // "Zero" means natural stack alignment.
736   if (A == 0)
737     A = HFI.getStackAlign().value();
738 
739   LLVM_DEBUG({
740     dbgs () << __func__ << " Align: " << A << " Size: ";
741     Size.getNode()->dump(&DAG);
742     dbgs() << "\n";
743   });
744 
745   SDValue AC = DAG.getConstant(A, dl, MVT::i32);
746   SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
747   SDValue AA = DAG.getNode(HexagonISD::ALLOCA, dl, VTs, Chain, Size, AC);
748 
749   DAG.ReplaceAllUsesOfValueWith(Op, AA);
750   return AA;
751 }
752 
753 SDValue HexagonTargetLowering::LowerFormalArguments(
754     SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
755     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
756     SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
757   MachineFunction &MF = DAG.getMachineFunction();
758   MachineFrameInfo &MFI = MF.getFrameInfo();
759   MachineRegisterInfo &MRI = MF.getRegInfo();
760 
761   // Linux ABI treats var-arg calls the same way as regular ones.
762   bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;
763 
764   // Assign locations to all of the incoming arguments.
765   SmallVector<CCValAssign, 16> ArgLocs;
766   HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs,
767                         *DAG.getContext(),
768                         MF.getFunction().getFunctionType()->getNumParams());
769 
770   if (Subtarget.useHVXOps())
771     CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_HVX);
772   else if (DisableArgsMinAlignment)
773     CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_Legacy);
774   else
775     CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon);
776 
777   // For LLVM, in the case when returning a struct by value (>8byte),
778   // the first argument is a pointer that points to the location on caller's
779   // stack where the return value will be stored. For Hexagon, the location on
780   // caller's stack is passed only when the struct size is smaller than (and
781   // equal to) 8 bytes. If not, no address will be passed into callee and
782   // callee return the result direclty through R0/R1.
783   auto NextSingleReg = [] (const TargetRegisterClass &RC, unsigned Reg) {
784     switch (RC.getID()) {
785     case Hexagon::IntRegsRegClassID:
786       return Reg - Hexagon::R0 + 1;
787     case Hexagon::DoubleRegsRegClassID:
788       return (Reg - Hexagon::D0 + 1) * 2;
789     case Hexagon::HvxVRRegClassID:
790       return Reg - Hexagon::V0 + 1;
791     case Hexagon::HvxWRRegClassID:
792       return (Reg - Hexagon::W0 + 1) * 2;
793     }
794     llvm_unreachable("Unexpected register class");
795   };
796 
797   auto &HFL = const_cast<HexagonFrameLowering&>(*Subtarget.getFrameLowering());
798   auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();
799   HFL.FirstVarArgSavedReg = 0;
800   HMFI.setFirstNamedArgFrameIndex(-int(MFI.getNumFixedObjects()));
801 
802   for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
803     CCValAssign &VA = ArgLocs[i];
804     ISD::ArgFlagsTy Flags = Ins[i].Flags;
805     bool ByVal = Flags.isByVal();
806 
807     // Arguments passed in registers:
808     // 1. 32- and 64-bit values and HVX vectors are passed directly,
809     // 2. Large structs are passed via an address, and the address is
810     //    passed in a register.
811     if (VA.isRegLoc() && ByVal && Flags.getByValSize() <= 8)
812       llvm_unreachable("ByValSize must be bigger than 8 bytes");
813 
814     bool InReg = VA.isRegLoc() &&
815                  (!ByVal || (ByVal && Flags.getByValSize() > 8));
816 
817     if (InReg) {
818       MVT RegVT = VA.getLocVT();
819       if (VA.getLocInfo() == CCValAssign::BCvt)
820         RegVT = VA.getValVT();
821 
822       const TargetRegisterClass *RC = getRegClassFor(RegVT);
823       Register VReg = MRI.createVirtualRegister(RC);
824       SDValue Copy = DAG.getCopyFromReg(Chain, dl, VReg, RegVT);
825 
826       // Treat values of type MVT::i1 specially: they are passed in
827       // registers of type i32, but they need to remain as values of
828       // type i1 for consistency of the argument lowering.
829       if (VA.getValVT() == MVT::i1) {
830         assert(RegVT.getSizeInBits() <= 32);
831         SDValue T = DAG.getNode(ISD::AND, dl, RegVT,
832                                 Copy, DAG.getConstant(1, dl, RegVT));
833         Copy = DAG.getSetCC(dl, MVT::i1, T, DAG.getConstant(0, dl, RegVT),
834                             ISD::SETNE);
835       } else {
836 #ifndef NDEBUG
837         unsigned RegSize = RegVT.getSizeInBits();
838         assert(RegSize == 32 || RegSize == 64 ||
839                Subtarget.isHVXVectorType(RegVT));
840 #endif
841       }
842       InVals.push_back(Copy);
843       MRI.addLiveIn(VA.getLocReg(), VReg);
844       HFL.FirstVarArgSavedReg = NextSingleReg(*RC, VA.getLocReg());
845     } else {
846       assert(VA.isMemLoc() && "Argument should be passed in memory");
847 
848       // If it's a byval parameter, then we need to compute the
849       // "real" size, not the size of the pointer.
850       unsigned ObjSize = Flags.isByVal()
851                             ? Flags.getByValSize()
852                             : VA.getLocVT().getStoreSizeInBits() / 8;
853 
854       // Create the frame index object for this incoming parameter.
855       int Offset = HEXAGON_LRFP_SIZE + VA.getLocMemOffset();
856       int FI = MFI.CreateFixedObject(ObjSize, Offset, true);
857       SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
858 
859       if (Flags.isByVal()) {
860         // If it's a pass-by-value aggregate, then do not dereference the stack
861         // location. Instead, we should generate a reference to the stack
862         // location.
863         InVals.push_back(FIN);
864       } else {
865         SDValue L = DAG.getLoad(VA.getValVT(), dl, Chain, FIN,
866                                 MachinePointerInfo::getFixedStack(MF, FI, 0));
867         InVals.push_back(L);
868       }
869     }
870   }
871 
872   if (IsVarArg && Subtarget.isEnvironmentMusl()) {
873     for (int i = HFL.FirstVarArgSavedReg; i < 6; i++)
874       MRI.addLiveIn(Hexagon::R0+i);
875   }
876 
877   if (IsVarArg && Subtarget.isEnvironmentMusl()) {
878     HMFI.setFirstNamedArgFrameIndex(HMFI.getFirstNamedArgFrameIndex() - 1);
879     HMFI.setLastNamedArgFrameIndex(-int(MFI.getNumFixedObjects()));
880 
881     // Create Frame index for the start of register saved area.
882     int NumVarArgRegs = 6 - HFL.FirstVarArgSavedReg;
883     bool RequiresPadding = (NumVarArgRegs & 1);
884     int RegSaveAreaSizePlusPadding = RequiresPadding
885                                         ? (NumVarArgRegs + 1) * 4
886                                         : NumVarArgRegs * 4;
887 
888     if (RegSaveAreaSizePlusPadding > 0) {
889       // The offset to saved register area should be 8 byte aligned.
890       int RegAreaStart = HEXAGON_LRFP_SIZE + CCInfo.getNextStackOffset();
891       if (!(RegAreaStart % 8))
892         RegAreaStart = (RegAreaStart + 7) & -8;
893 
894       int RegSaveAreaFrameIndex =
895         MFI.CreateFixedObject(RegSaveAreaSizePlusPadding, RegAreaStart, true);
896       HMFI.setRegSavedAreaStartFrameIndex(RegSaveAreaFrameIndex);
897 
898       // This will point to the next argument passed via stack.
899       int Offset = RegAreaStart + RegSaveAreaSizePlusPadding;
900       int FI = MFI.CreateFixedObject(Hexagon_PointerSize, Offset, true);
901       HMFI.setVarArgsFrameIndex(FI);
902     } else {
903       // This will point to the next argument passed via stack, when
904       // there is no saved register area.
905       int Offset = HEXAGON_LRFP_SIZE + CCInfo.getNextStackOffset();
906       int FI = MFI.CreateFixedObject(Hexagon_PointerSize, Offset, true);
907       HMFI.setRegSavedAreaStartFrameIndex(FI);
908       HMFI.setVarArgsFrameIndex(FI);
909     }
910   }
911 
912 
913   if (IsVarArg && !Subtarget.isEnvironmentMusl()) {
914     // This will point to the next argument passed via stack.
915     int Offset = HEXAGON_LRFP_SIZE + CCInfo.getNextStackOffset();
916     int FI = MFI.CreateFixedObject(Hexagon_PointerSize, Offset, true);
917     HMFI.setVarArgsFrameIndex(FI);
918   }
919 
920   return Chain;
921 }
922 
923 SDValue
924 HexagonTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
925   // VASTART stores the address of the VarArgsFrameIndex slot into the
926   // memory location argument.
927   MachineFunction &MF = DAG.getMachineFunction();
928   HexagonMachineFunctionInfo *QFI = MF.getInfo<HexagonMachineFunctionInfo>();
929   SDValue Addr = DAG.getFrameIndex(QFI->getVarArgsFrameIndex(), MVT::i32);
930   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
931 
932   if (!Subtarget.isEnvironmentMusl()) {
933     return DAG.getStore(Op.getOperand(0), SDLoc(Op), Addr, Op.getOperand(1),
934                         MachinePointerInfo(SV));
935   }
936   auto &FuncInfo = *MF.getInfo<HexagonMachineFunctionInfo>();
937   auto &HFL = *Subtarget.getFrameLowering();
938   SDLoc DL(Op);
939   SmallVector<SDValue, 8> MemOps;
940 
941   // Get frame index of va_list.
942   SDValue FIN = Op.getOperand(1);
943 
944   // If first Vararg register is odd, add 4 bytes to start of
945   // saved register area to point to the first register location.
946   // This is because the saved register area has to be 8 byte aligned.
947   // Incase of an odd start register, there will be 4 bytes of padding in
948   // the beginning of saved register area. If all registers area used up,
949   // the following condition will handle it correctly.
950   SDValue SavedRegAreaStartFrameIndex =
951     DAG.getFrameIndex(FuncInfo.getRegSavedAreaStartFrameIndex(), MVT::i32);
952 
953   auto PtrVT = getPointerTy(DAG.getDataLayout());
954 
955   if (HFL.FirstVarArgSavedReg & 1)
956     SavedRegAreaStartFrameIndex =
957       DAG.getNode(ISD::ADD, DL, PtrVT,
958                   DAG.getFrameIndex(FuncInfo.getRegSavedAreaStartFrameIndex(),
959                                     MVT::i32),
960                   DAG.getIntPtrConstant(4, DL));
961 
962   // Store the saved register area start pointer.
963   SDValue Store =
964     DAG.getStore(Op.getOperand(0), DL,
965                  SavedRegAreaStartFrameIndex,
966                  FIN, MachinePointerInfo(SV));
967   MemOps.push_back(Store);
968 
969   // Store saved register area end pointer.
970   FIN = DAG.getNode(ISD::ADD, DL, PtrVT,
971                     FIN, DAG.getIntPtrConstant(4, DL));
972   Store = DAG.getStore(Op.getOperand(0), DL,
973                        DAG.getFrameIndex(FuncInfo.getVarArgsFrameIndex(),
974                                          PtrVT),
975                        FIN, MachinePointerInfo(SV, 4));
976   MemOps.push_back(Store);
977 
978   // Store overflow area pointer.
979   FIN = DAG.getNode(ISD::ADD, DL, PtrVT,
980                     FIN, DAG.getIntPtrConstant(4, DL));
981   Store = DAG.getStore(Op.getOperand(0), DL,
982                        DAG.getFrameIndex(FuncInfo.getVarArgsFrameIndex(),
983                                          PtrVT),
984                        FIN, MachinePointerInfo(SV, 8));
985   MemOps.push_back(Store);
986 
987   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
988 }
989 
990 SDValue
991 HexagonTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
992   // Assert that the linux ABI is enabled for the current compilation.
993   assert(Subtarget.isEnvironmentMusl() && "Linux ABI should be enabled");
994   SDValue Chain = Op.getOperand(0);
995   SDValue DestPtr = Op.getOperand(1);
996   SDValue SrcPtr = Op.getOperand(2);
997   const Value *DestSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
998   const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
999   SDLoc DL(Op);
1000   // Size of the va_list is 12 bytes as it has 3 pointers. Therefore,
1001   // we need to memcopy 12 bytes from va_list to another similar list.
1002   return DAG.getMemcpy(Chain, DL, DestPtr, SrcPtr,
1003                        DAG.getIntPtrConstant(12, DL), Align(4),
1004                        /*isVolatile*/ false, false, false,
1005                        MachinePointerInfo(DestSV), MachinePointerInfo(SrcSV));
1006 }
1007 
1008 SDValue HexagonTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
1009   const SDLoc &dl(Op);
1010   SDValue LHS = Op.getOperand(0);
1011   SDValue RHS = Op.getOperand(1);
1012   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
1013   MVT ResTy = ty(Op);
1014   MVT OpTy = ty(LHS);
1015 
1016   if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {
1017     MVT ElemTy = OpTy.getVectorElementType();
1018     assert(ElemTy.isScalarInteger());
1019     MVT WideTy = MVT::getVectorVT(MVT::getIntegerVT(2*ElemTy.getSizeInBits()),
1020                                   OpTy.getVectorNumElements());
1021     return DAG.getSetCC(dl, ResTy,
1022                         DAG.getSExtOrTrunc(LHS, SDLoc(LHS), WideTy),
1023                         DAG.getSExtOrTrunc(RHS, SDLoc(RHS), WideTy), CC);
1024   }
1025 
1026   // Treat all other vector types as legal.
1027   if (ResTy.isVector())
1028     return Op;
1029 
1030   // Comparisons of short integers should use sign-extend, not zero-extend,
1031   // since we can represent small negative values in the compare instructions.
1032   // The LLVM default is to use zero-extend arbitrarily in these cases.
1033   auto isSExtFree = [this](SDValue N) {
1034     switch (N.getOpcode()) {
1035       case ISD::TRUNCATE: {
1036         // A sign-extend of a truncate of a sign-extend is free.
1037         SDValue Op = N.getOperand(0);
1038         if (Op.getOpcode() != ISD::AssertSext)
1039           return false;
1040         EVT OrigTy = cast<VTSDNode>(Op.getOperand(1))->getVT();
1041         unsigned ThisBW = ty(N).getSizeInBits();
1042         unsigned OrigBW = OrigTy.getSizeInBits();
1043         // The type that was sign-extended to get the AssertSext must be
1044         // narrower than the type of N (so that N has still the same value
1045         // as the original).
1046         return ThisBW >= OrigBW;
1047       }
1048       case ISD::LOAD:
1049         // We have sign-extended loads.
1050         return true;
1051     }
1052     return false;
1053   };
1054 
1055   if (OpTy == MVT::i8 || OpTy == MVT::i16) {
1056     ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS);
1057     bool IsNegative = C && C->getAPIntValue().isNegative();
1058     if (IsNegative || isSExtFree(LHS) || isSExtFree(RHS))
1059       return DAG.getSetCC(dl, ResTy,
1060                           DAG.getSExtOrTrunc(LHS, SDLoc(LHS), MVT::i32),
1061                           DAG.getSExtOrTrunc(RHS, SDLoc(RHS), MVT::i32), CC);
1062   }
1063 
1064   return SDValue();
1065 }
1066 
1067 SDValue
1068 HexagonTargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
1069   SDValue PredOp = Op.getOperand(0);
1070   SDValue Op1 = Op.getOperand(1), Op2 = Op.getOperand(2);
1071   MVT OpTy = ty(Op1);
1072   const SDLoc &dl(Op);
1073 
1074   if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {
1075     MVT ElemTy = OpTy.getVectorElementType();
1076     assert(ElemTy.isScalarInteger());
1077     MVT WideTy = MVT::getVectorVT(MVT::getIntegerVT(2*ElemTy.getSizeInBits()),
1078                                   OpTy.getVectorNumElements());
1079     // Generate (trunc (select (_, sext, sext))).
1080     return DAG.getSExtOrTrunc(
1081               DAG.getSelect(dl, WideTy, PredOp,
1082                             DAG.getSExtOrTrunc(Op1, dl, WideTy),
1083                             DAG.getSExtOrTrunc(Op2, dl, WideTy)),
1084               dl, OpTy);
1085   }
1086 
1087   return SDValue();
1088 }
1089 
1090 SDValue
1091 HexagonTargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
1092   EVT ValTy = Op.getValueType();
1093   ConstantPoolSDNode *CPN = cast<ConstantPoolSDNode>(Op);
1094   Constant *CVal = nullptr;
1095   bool isVTi1Type = false;
1096   if (auto *CV = dyn_cast<ConstantVector>(CPN->getConstVal())) {
1097     if (cast<VectorType>(CV->getType())->getElementType()->isIntegerTy(1)) {
1098       IRBuilder<> IRB(CV->getContext());
1099       SmallVector<Constant*, 128> NewConst;
1100       unsigned VecLen = CV->getNumOperands();
1101       assert(isPowerOf2_32(VecLen) &&
1102              "conversion only supported for pow2 VectorSize");
1103       for (unsigned i = 0; i < VecLen; ++i)
1104         NewConst.push_back(IRB.getInt8(CV->getOperand(i)->isZeroValue()));
1105 
1106       CVal = ConstantVector::get(NewConst);
1107       isVTi1Type = true;
1108     }
1109   }
1110   Align Alignment = CPN->getAlign();
1111   bool IsPositionIndependent = isPositionIndependent();
1112   unsigned char TF = IsPositionIndependent ? HexagonII::MO_PCREL : 0;
1113 
1114   unsigned Offset = 0;
1115   SDValue T;
1116   if (CPN->isMachineConstantPoolEntry())
1117     T = DAG.getTargetConstantPool(CPN->getMachineCPVal(), ValTy, Alignment,
1118                                   Offset, TF);
1119   else if (isVTi1Type)
1120     T = DAG.getTargetConstantPool(CVal, ValTy, Alignment, Offset, TF);
1121   else
1122     T = DAG.getTargetConstantPool(CPN->getConstVal(), ValTy, Alignment, Offset,
1123                                   TF);
1124 
1125   assert(cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF &&
1126          "Inconsistent target flag encountered");
1127 
1128   if (IsPositionIndependent)
1129     return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), ValTy, T);
1130   return DAG.getNode(HexagonISD::CP, SDLoc(Op), ValTy, T);
1131 }
1132 
1133 SDValue
1134 HexagonTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
1135   EVT VT = Op.getValueType();
1136   int Idx = cast<JumpTableSDNode>(Op)->getIndex();
1137   if (isPositionIndependent()) {
1138     SDValue T = DAG.getTargetJumpTable(Idx, VT, HexagonII::MO_PCREL);
1139     return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), VT, T);
1140   }
1141 
1142   SDValue T = DAG.getTargetJumpTable(Idx, VT);
1143   return DAG.getNode(HexagonISD::JT, SDLoc(Op), VT, T);
1144 }
1145 
1146 SDValue
1147 HexagonTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
1148   const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
1149   MachineFunction &MF = DAG.getMachineFunction();
1150   MachineFrameInfo &MFI = MF.getFrameInfo();
1151   MFI.setReturnAddressIsTaken(true);
1152 
1153   if (verifyReturnAddressArgumentIsConstant(Op, DAG))
1154     return SDValue();
1155 
1156   EVT VT = Op.getValueType();
1157   SDLoc dl(Op);
1158   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1159   if (Depth) {
1160     SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
1161     SDValue Offset = DAG.getConstant(4, dl, MVT::i32);
1162     return DAG.getLoad(VT, dl, DAG.getEntryNode(),
1163                        DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset),
1164                        MachinePointerInfo());
1165   }
1166 
1167   // Return LR, which contains the return address. Mark it an implicit live-in.
1168   unsigned Reg = MF.addLiveIn(HRI.getRARegister(), getRegClassFor(MVT::i32));
1169   return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);
1170 }
1171 
1172 SDValue
1173 HexagonTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
1174   const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
1175   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
1176   MFI.setFrameAddressIsTaken(true);
1177 
1178   EVT VT = Op.getValueType();
1179   SDLoc dl(Op);
1180   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1181   SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl,
1182                                          HRI.getFrameRegister(), VT);
1183   while (Depth--)
1184     FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
1185                             MachinePointerInfo());
1186   return FrameAddr;
1187 }
1188 
1189 SDValue
1190 HexagonTargetLowering::LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const {
1191   SDLoc dl(Op);
1192   return DAG.getNode(HexagonISD::BARRIER, dl, MVT::Other, Op.getOperand(0));
1193 }
1194 
1195 SDValue
1196 HexagonTargetLowering::LowerGLOBALADDRESS(SDValue Op, SelectionDAG &DAG) const {
1197   SDLoc dl(Op);
1198   auto *GAN = cast<GlobalAddressSDNode>(Op);
1199   auto PtrVT = getPointerTy(DAG.getDataLayout());
1200   auto *GV = GAN->getGlobal();
1201   int64_t Offset = GAN->getOffset();
1202 
1203   auto &HLOF = *HTM.getObjFileLowering();
1204   Reloc::Model RM = HTM.getRelocationModel();
1205 
1206   if (RM == Reloc::Static) {
1207     SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset);
1208     const GlobalObject *GO = GV->getBaseObject();
1209     if (GO && Subtarget.useSmallData() && HLOF.isGlobalInSmallSection(GO, HTM))
1210       return DAG.getNode(HexagonISD::CONST32_GP, dl, PtrVT, GA);
1211     return DAG.getNode(HexagonISD::CONST32, dl, PtrVT, GA);
1212   }
1213 
1214   bool UsePCRel = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV);
1215   if (UsePCRel) {
1216     SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset,
1217                                             HexagonII::MO_PCREL);
1218     return DAG.getNode(HexagonISD::AT_PCREL, dl, PtrVT, GA);
1219   }
1220 
1221   // Use GOT index.
1222   SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(PtrVT);
1223   SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, HexagonII::MO_GOT);
1224   SDValue Off = DAG.getConstant(Offset, dl, MVT::i32);
1225   return DAG.getNode(HexagonISD::AT_GOT, dl, PtrVT, GOT, GA, Off);
1226 }
1227 
1228 // Specifies that for loads and stores VT can be promoted to PromotedLdStVT.
1229 SDValue
1230 HexagonTargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
1231   const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
1232   SDLoc dl(Op);
1233   EVT PtrVT = getPointerTy(DAG.getDataLayout());
1234 
1235   Reloc::Model RM = HTM.getRelocationModel();
1236   if (RM == Reloc::Static) {
1237     SDValue A = DAG.getTargetBlockAddress(BA, PtrVT);
1238     return DAG.getNode(HexagonISD::CONST32_GP, dl, PtrVT, A);
1239   }
1240 
1241   SDValue A = DAG.getTargetBlockAddress(BA, PtrVT, 0, HexagonII::MO_PCREL);
1242   return DAG.getNode(HexagonISD::AT_PCREL, dl, PtrVT, A);
1243 }
1244 
1245 SDValue
1246 HexagonTargetLowering::LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG)
1247       const {
1248   EVT PtrVT = getPointerTy(DAG.getDataLayout());
1249   SDValue GOTSym = DAG.getTargetExternalSymbol(HEXAGON_GOT_SYM_NAME, PtrVT,
1250                                                HexagonII::MO_PCREL);
1251   return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), PtrVT, GOTSym);
1252 }
1253 
1254 SDValue
1255 HexagonTargetLowering::GetDynamicTLSAddr(SelectionDAG &DAG, SDValue Chain,
1256       GlobalAddressSDNode *GA, SDValue Glue, EVT PtrVT, unsigned ReturnReg,
1257       unsigned char OperandFlags) const {
1258   MachineFunction &MF = DAG.getMachineFunction();
1259   MachineFrameInfo &MFI = MF.getFrameInfo();
1260   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
1261   SDLoc dl(GA);
1262   SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
1263                                            GA->getValueType(0),
1264                                            GA->getOffset(),
1265                                            OperandFlags);
1266   // Create Operands for the call.The Operands should have the following:
1267   // 1. Chain SDValue
1268   // 2. Callee which in this case is the Global address value.
1269   // 3. Registers live into the call.In this case its R0, as we
1270   //    have just one argument to be passed.
1271   // 4. Glue.
1272   // Note: The order is important.
1273 
1274   const auto &HRI = *Subtarget.getRegisterInfo();
1275   const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallingConv::C);
1276   assert(Mask && "Missing call preserved mask for calling convention");
1277   SDValue Ops[] = { Chain, TGA, DAG.getRegister(Hexagon::R0, PtrVT),
1278                     DAG.getRegisterMask(Mask), Glue };
1279   Chain = DAG.getNode(HexagonISD::CALL, dl, NodeTys, Ops);
1280 
1281   // Inform MFI that function has calls.
1282   MFI.setAdjustsStack(true);
1283 
1284   Glue = Chain.getValue(1);
1285   return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Glue);
1286 }
1287 
1288 //
1289 // Lower using the intial executable model for TLS addresses
1290 //
1291 SDValue
1292 HexagonTargetLowering::LowerToTLSInitialExecModel(GlobalAddressSDNode *GA,
1293       SelectionDAG &DAG) const {
1294   SDLoc dl(GA);
1295   int64_t Offset = GA->getOffset();
1296   auto PtrVT = getPointerTy(DAG.getDataLayout());
1297 
1298   // Get the thread pointer.
1299   SDValue TP = DAG.getCopyFromReg(DAG.getEntryNode(), dl, Hexagon::UGP, PtrVT);
1300 
1301   bool IsPositionIndependent = isPositionIndependent();
1302   unsigned char TF =
1303       IsPositionIndependent ? HexagonII::MO_IEGOT : HexagonII::MO_IE;
1304 
1305   // First generate the TLS symbol address
1306   SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT,
1307                                            Offset, TF);
1308 
1309   SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA);
1310 
1311   if (IsPositionIndependent) {
1312     // Generate the GOT pointer in case of position independent code
1313     SDValue GOT = LowerGLOBAL_OFFSET_TABLE(Sym, DAG);
1314 
1315     // Add the TLS Symbol address to GOT pointer.This gives
1316     // GOT relative relocation for the symbol.
1317     Sym = DAG.getNode(ISD::ADD, dl, PtrVT, GOT, Sym);
1318   }
1319 
1320   // Load the offset value for TLS symbol.This offset is relative to
1321   // thread pointer.
1322   SDValue LoadOffset =
1323       DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Sym, MachinePointerInfo());
1324 
1325   // Address of the thread local variable is the add of thread
1326   // pointer and the offset of the variable.
1327   return DAG.getNode(ISD::ADD, dl, PtrVT, TP, LoadOffset);
1328 }
1329 
1330 //
1331 // Lower using the local executable model for TLS addresses
1332 //
1333 SDValue
1334 HexagonTargetLowering::LowerToTLSLocalExecModel(GlobalAddressSDNode *GA,
1335       SelectionDAG &DAG) const {
1336   SDLoc dl(GA);
1337   int64_t Offset = GA->getOffset();
1338   auto PtrVT = getPointerTy(DAG.getDataLayout());
1339 
1340   // Get the thread pointer.
1341   SDValue TP = DAG.getCopyFromReg(DAG.getEntryNode(), dl, Hexagon::UGP, PtrVT);
1342   // Generate the TLS symbol address
1343   SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, Offset,
1344                                            HexagonII::MO_TPREL);
1345   SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA);
1346 
1347   // Address of the thread local variable is the add of thread
1348   // pointer and the offset of the variable.
1349   return DAG.getNode(ISD::ADD, dl, PtrVT, TP, Sym);
1350 }
1351 
1352 //
1353 // Lower using the general dynamic model for TLS addresses
1354 //
1355 SDValue
1356 HexagonTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
1357       SelectionDAG &DAG) const {
1358   SDLoc dl(GA);
1359   int64_t Offset = GA->getOffset();
1360   auto PtrVT = getPointerTy(DAG.getDataLayout());
1361 
1362   // First generate the TLS symbol address
1363   SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, Offset,
1364                                            HexagonII::MO_GDGOT);
1365 
1366   // Then, generate the GOT pointer
1367   SDValue GOT = LowerGLOBAL_OFFSET_TABLE(TGA, DAG);
1368 
1369   // Add the TLS symbol and the GOT pointer
1370   SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA);
1371   SDValue Chain = DAG.getNode(ISD::ADD, dl, PtrVT, GOT, Sym);
1372 
1373   // Copy over the argument to R0
1374   SDValue InFlag;
1375   Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, Hexagon::R0, Chain, InFlag);
1376   InFlag = Chain.getValue(1);
1377 
1378   unsigned Flags =
1379       static_cast<const HexagonSubtarget &>(DAG.getSubtarget()).useLongCalls()
1380           ? HexagonII::MO_GDPLT | HexagonII::HMOTF_ConstExtended
1381           : HexagonII::MO_GDPLT;
1382 
1383   return GetDynamicTLSAddr(DAG, Chain, GA, InFlag, PtrVT,
1384                            Hexagon::R0, Flags);
1385 }
1386 
1387 //
1388 // Lower TLS addresses.
1389 //
1390 // For now for dynamic models, we only support the general dynamic model.
1391 //
1392 SDValue
1393 HexagonTargetLowering::LowerGlobalTLSAddress(SDValue Op,
1394       SelectionDAG &DAG) const {
1395   GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
1396 
1397   switch (HTM.getTLSModel(GA->getGlobal())) {
1398     case TLSModel::GeneralDynamic:
1399     case TLSModel::LocalDynamic:
1400       return LowerToTLSGeneralDynamicModel(GA, DAG);
1401     case TLSModel::InitialExec:
1402       return LowerToTLSInitialExecModel(GA, DAG);
1403     case TLSModel::LocalExec:
1404       return LowerToTLSLocalExecModel(GA, DAG);
1405   }
1406   llvm_unreachable("Bogus TLS model");
1407 }
1408 
1409 //===----------------------------------------------------------------------===//
1410 // TargetLowering Implementation
1411 //===----------------------------------------------------------------------===//
1412 
1413 HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
1414                                              const HexagonSubtarget &ST)
1415     : TargetLowering(TM), HTM(static_cast<const HexagonTargetMachine&>(TM)),
1416       Subtarget(ST) {
1417   auto &HRI = *Subtarget.getRegisterInfo();
1418 
1419   setPrefLoopAlignment(Align(16));
1420   setMinFunctionAlignment(Align(4));
1421   setPrefFunctionAlignment(Align(16));
1422   setStackPointerRegisterToSaveRestore(HRI.getStackRegister());
1423   setBooleanContents(TargetLoweringBase::UndefinedBooleanContent);
1424   setBooleanVectorContents(TargetLoweringBase::UndefinedBooleanContent);
1425 
1426   setMaxAtomicSizeInBitsSupported(64);
1427   setMinCmpXchgSizeInBits(32);
1428 
1429   if (EnableHexSDNodeSched)
1430     setSchedulingPreference(Sched::VLIW);
1431   else
1432     setSchedulingPreference(Sched::Source);
1433 
1434   // Limits for inline expansion of memcpy/memmove
1435   MaxStoresPerMemcpy = MaxStoresPerMemcpyCL;
1436   MaxStoresPerMemcpyOptSize = MaxStoresPerMemcpyOptSizeCL;
1437   MaxStoresPerMemmove = MaxStoresPerMemmoveCL;
1438   MaxStoresPerMemmoveOptSize = MaxStoresPerMemmoveOptSizeCL;
1439   MaxStoresPerMemset = MaxStoresPerMemsetCL;
1440   MaxStoresPerMemsetOptSize = MaxStoresPerMemsetOptSizeCL;
1441 
1442   //
1443   // Set up register classes.
1444   //
1445 
1446   addRegisterClass(MVT::i1,    &Hexagon::PredRegsRegClass);
1447   addRegisterClass(MVT::v2i1,  &Hexagon::PredRegsRegClass);  // bbbbaaaa
1448   addRegisterClass(MVT::v4i1,  &Hexagon::PredRegsRegClass);  // ddccbbaa
1449   addRegisterClass(MVT::v8i1,  &Hexagon::PredRegsRegClass);  // hgfedcba
1450   addRegisterClass(MVT::i32,   &Hexagon::IntRegsRegClass);
1451   addRegisterClass(MVT::v2i16, &Hexagon::IntRegsRegClass);
1452   addRegisterClass(MVT::v4i8,  &Hexagon::IntRegsRegClass);
1453   addRegisterClass(MVT::i64,   &Hexagon::DoubleRegsRegClass);
1454   addRegisterClass(MVT::v8i8,  &Hexagon::DoubleRegsRegClass);
1455   addRegisterClass(MVT::v4i16, &Hexagon::DoubleRegsRegClass);
1456   addRegisterClass(MVT::v2i32, &Hexagon::DoubleRegsRegClass);
1457 
1458   addRegisterClass(MVT::f32, &Hexagon::IntRegsRegClass);
1459   addRegisterClass(MVT::f64, &Hexagon::DoubleRegsRegClass);
1460 
1461   //
1462   // Handling of scalar operations.
1463   //
1464   // All operations default to "legal", except:
1465   // - indexed loads and stores (pre-/post-incremented),
1466   // - ANY_EXTEND_VECTOR_INREG, ATOMIC_CMP_SWAP_WITH_SUCCESS, CONCAT_VECTORS,
1467   //   ConstantFP, DEBUGTRAP, FCEIL, FCOPYSIGN, FEXP, FEXP2, FFLOOR, FGETSIGN,
1468   //   FLOG, FLOG2, FLOG10, FMAXNUM, FMINNUM, FNEARBYINT, FRINT, FROUND, TRAP,
1469   //   FTRUNC, PREFETCH, SIGN_EXTEND_VECTOR_INREG, ZERO_EXTEND_VECTOR_INREG,
1470   // which default to "expand" for at least one type.
1471 
1472   // Misc operations.
1473   setOperationAction(ISD::ConstantFP,           MVT::f32,   Legal);
1474   setOperationAction(ISD::ConstantFP,           MVT::f64,   Legal);
1475   setOperationAction(ISD::TRAP,                 MVT::Other, Legal);
1476   setOperationAction(ISD::ConstantPool,         MVT::i32,   Custom);
1477   setOperationAction(ISD::JumpTable,            MVT::i32,   Custom);
1478   setOperationAction(ISD::BUILD_PAIR,           MVT::i64,   Expand);
1479   setOperationAction(ISD::SIGN_EXTEND_INREG,    MVT::i1,    Expand);
1480   setOperationAction(ISD::INLINEASM,            MVT::Other, Custom);
1481   setOperationAction(ISD::INLINEASM_BR,         MVT::Other, Custom);
1482   setOperationAction(ISD::PREFETCH,             MVT::Other, Custom);
1483   setOperationAction(ISD::READCYCLECOUNTER,     MVT::i64,   Custom);
1484   setOperationAction(ISD::INTRINSIC_VOID,       MVT::Other, Custom);
1485   setOperationAction(ISD::EH_RETURN,            MVT::Other, Custom);
1486   setOperationAction(ISD::GLOBAL_OFFSET_TABLE,  MVT::i32,   Custom);
1487   setOperationAction(ISD::GlobalTLSAddress,     MVT::i32,   Custom);
1488   setOperationAction(ISD::ATOMIC_FENCE,         MVT::Other, Custom);
1489 
1490   // Custom legalize GlobalAddress nodes into CONST32.
1491   setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
1492   setOperationAction(ISD::GlobalAddress, MVT::i8,  Custom);
1493   setOperationAction(ISD::BlockAddress,  MVT::i32, Custom);
1494 
1495   // Hexagon needs to optimize cases with negative constants.
1496   setOperationAction(ISD::SETCC, MVT::i8,    Custom);
1497   setOperationAction(ISD::SETCC, MVT::i16,   Custom);
1498   setOperationAction(ISD::SETCC, MVT::v4i8,  Custom);
1499   setOperationAction(ISD::SETCC, MVT::v2i16, Custom);
1500 
1501   // VASTART needs to be custom lowered to use the VarArgsFrameIndex.
1502   setOperationAction(ISD::VASTART, MVT::Other, Custom);
1503   setOperationAction(ISD::VAEND,   MVT::Other, Expand);
1504   setOperationAction(ISD::VAARG,   MVT::Other, Expand);
1505   if (Subtarget.isEnvironmentMusl())
1506     setOperationAction(ISD::VACOPY, MVT::Other, Custom);
1507   else
1508     setOperationAction(ISD::VACOPY,  MVT::Other, Expand);
1509 
1510   setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
1511   setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
1512   setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom);
1513 
1514   if (EmitJumpTables)
1515     setMinimumJumpTableEntries(MinimumJumpTables);
1516   else
1517     setMinimumJumpTableEntries(std::numeric_limits<unsigned>::max());
1518   setOperationAction(ISD::BR_JT, MVT::Other, Expand);
1519 
1520   for (unsigned LegalIntOp :
1521        {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) {
1522     setOperationAction(LegalIntOp, MVT::i32, Legal);
1523     setOperationAction(LegalIntOp, MVT::i64, Legal);
1524   }
1525 
1526   // Hexagon has A4_addp_c and A4_subp_c that take and generate a carry bit,
1527   // but they only operate on i64.
1528   for (MVT VT : MVT::integer_valuetypes()) {
1529     setOperationAction(ISD::UADDO,    VT, Custom);
1530     setOperationAction(ISD::USUBO,    VT, Custom);
1531     setOperationAction(ISD::SADDO,    VT, Expand);
1532     setOperationAction(ISD::SSUBO,    VT, Expand);
1533     setOperationAction(ISD::ADDCARRY, VT, Expand);
1534     setOperationAction(ISD::SUBCARRY, VT, Expand);
1535   }
1536   setOperationAction(ISD::ADDCARRY, MVT::i64, Custom);
1537   setOperationAction(ISD::SUBCARRY, MVT::i64, Custom);
1538 
1539   setOperationAction(ISD::CTLZ, MVT::i8,  Promote);
1540   setOperationAction(ISD::CTLZ, MVT::i16, Promote);
1541   setOperationAction(ISD::CTTZ, MVT::i8,  Promote);
1542   setOperationAction(ISD::CTTZ, MVT::i16, Promote);
1543 
1544   // Popcount can count # of 1s in i64 but returns i32.
1545   setOperationAction(ISD::CTPOP, MVT::i8,  Promote);
1546   setOperationAction(ISD::CTPOP, MVT::i16, Promote);
1547   setOperationAction(ISD::CTPOP, MVT::i32, Promote);
1548   setOperationAction(ISD::CTPOP, MVT::i64, Legal);
1549 
1550   setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
1551   setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);
1552   setOperationAction(ISD::BSWAP, MVT::i32, Legal);
1553   setOperationAction(ISD::BSWAP, MVT::i64, Legal);
1554 
1555   setOperationAction(ISD::FSHL, MVT::i32, Legal);
1556   setOperationAction(ISD::FSHL, MVT::i64, Legal);
1557   setOperationAction(ISD::FSHR, MVT::i32, Legal);
1558   setOperationAction(ISD::FSHR, MVT::i64, Legal);
1559 
1560   for (unsigned IntExpOp :
1561        {ISD::SDIV,      ISD::UDIV,      ISD::SREM,      ISD::UREM,
1562         ISD::SDIVREM,   ISD::UDIVREM,   ISD::ROTL,      ISD::ROTR,
1563         ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS,
1564         ISD::SMUL_LOHI, ISD::UMUL_LOHI}) {
1565     for (MVT VT : MVT::integer_valuetypes())
1566       setOperationAction(IntExpOp, VT, Expand);
1567   }
1568 
1569   for (unsigned FPExpOp :
1570        {ISD::FDIV, ISD::FREM, ISD::FSQRT, ISD::FSIN, ISD::FCOS, ISD::FSINCOS,
1571         ISD::FPOW, ISD::FCOPYSIGN}) {
1572     for (MVT VT : MVT::fp_valuetypes())
1573       setOperationAction(FPExpOp, VT, Expand);
1574   }
1575 
1576   // No extending loads from i32.
1577   for (MVT VT : MVT::integer_valuetypes()) {
1578     setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i32, Expand);
1579     setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i32, Expand);
1580     setLoadExtAction(ISD::EXTLOAD,  VT, MVT::i32, Expand);
1581   }
1582   // Turn FP truncstore into trunc + store.
1583   setTruncStoreAction(MVT::f64, MVT::f32, Expand);
1584   // Turn FP extload into load/fpextend.
1585   for (MVT VT : MVT::fp_valuetypes())
1586     setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
1587 
1588   // Expand BR_CC and SELECT_CC for all integer and fp types.
1589   for (MVT VT : MVT::integer_valuetypes()) {
1590     setOperationAction(ISD::BR_CC,     VT, Expand);
1591     setOperationAction(ISD::SELECT_CC, VT, Expand);
1592   }
1593   for (MVT VT : MVT::fp_valuetypes()) {
1594     setOperationAction(ISD::BR_CC,     VT, Expand);
1595     setOperationAction(ISD::SELECT_CC, VT, Expand);
1596   }
1597   setOperationAction(ISD::BR_CC, MVT::Other, Expand);
1598 
1599   //
1600   // Handling of vector operations.
1601   //
1602 
1603   // Set the action for vector operations to "expand", then override it with
1604   // either "custom" or "legal" for specific cases.
1605   static const unsigned VectExpOps[] = {
1606     // Integer arithmetic:
1607     ISD::ADD,     ISD::SUB,     ISD::MUL,     ISD::SDIV,      ISD::UDIV,
1608     ISD::SREM,    ISD::UREM,    ISD::SDIVREM, ISD::UDIVREM,   ISD::SADDO,
1609     ISD::UADDO,   ISD::SSUBO,   ISD::USUBO,   ISD::SMUL_LOHI, ISD::UMUL_LOHI,
1610     // Logical/bit:
1611     ISD::AND,     ISD::OR,      ISD::XOR,     ISD::ROTL,    ISD::ROTR,
1612     ISD::CTPOP,   ISD::CTLZ,    ISD::CTTZ,
1613     // Floating point arithmetic/math functions:
1614     ISD::FADD,    ISD::FSUB,    ISD::FMUL,    ISD::FMA,     ISD::FDIV,
1615     ISD::FREM,    ISD::FNEG,    ISD::FABS,    ISD::FSQRT,   ISD::FSIN,
1616     ISD::FCOS,    ISD::FPOW,    ISD::FLOG,    ISD::FLOG2,
1617     ISD::FLOG10,  ISD::FEXP,    ISD::FEXP2,   ISD::FCEIL,   ISD::FTRUNC,
1618     ISD::FRINT,   ISD::FNEARBYINT,            ISD::FROUND,  ISD::FFLOOR,
1619     ISD::FMINNUM, ISD::FMAXNUM, ISD::FSINCOS,
1620     // Misc:
1621     ISD::BR_CC,   ISD::SELECT_CC,             ISD::ConstantPool,
1622     // Vector:
1623     ISD::BUILD_VECTOR,          ISD::SCALAR_TO_VECTOR,
1624     ISD::EXTRACT_VECTOR_ELT,    ISD::INSERT_VECTOR_ELT,
1625     ISD::EXTRACT_SUBVECTOR,     ISD::INSERT_SUBVECTOR,
1626     ISD::CONCAT_VECTORS,        ISD::VECTOR_SHUFFLE,
1627     ISD::SPLAT_VECTOR,
1628   };
1629 
1630   for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
1631     for (unsigned VectExpOp : VectExpOps)
1632       setOperationAction(VectExpOp, VT, Expand);
1633 
1634     // Expand all extending loads and truncating stores:
1635     for (MVT TargetVT : MVT::fixedlen_vector_valuetypes()) {
1636       if (TargetVT == VT)
1637         continue;
1638       setLoadExtAction(ISD::EXTLOAD, TargetVT, VT, Expand);
1639       setLoadExtAction(ISD::ZEXTLOAD, TargetVT, VT, Expand);
1640       setLoadExtAction(ISD::SEXTLOAD, TargetVT, VT, Expand);
1641       setTruncStoreAction(VT, TargetVT, Expand);
1642     }
1643 
1644     // Normalize all inputs to SELECT to be vectors of i32.
1645     if (VT.getVectorElementType() != MVT::i32) {
1646       MVT VT32 = MVT::getVectorVT(MVT::i32, VT.getSizeInBits()/32);
1647       setOperationAction(ISD::SELECT, VT, Promote);
1648       AddPromotedToType(ISD::SELECT, VT, VT32);
1649     }
1650     setOperationAction(ISD::SRA, VT, Custom);
1651     setOperationAction(ISD::SHL, VT, Custom);
1652     setOperationAction(ISD::SRL, VT, Custom);
1653   }
1654 
1655   // Extending loads from (native) vectors of i8 into (native) vectors of i16
1656   // are legal.
1657   setLoadExtAction(ISD::EXTLOAD,  MVT::v2i16, MVT::v2i8, Legal);
1658   setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i16, MVT::v2i8, Legal);
1659   setLoadExtAction(ISD::SEXTLOAD, MVT::v2i16, MVT::v2i8, Legal);
1660   setLoadExtAction(ISD::EXTLOAD,  MVT::v4i16, MVT::v4i8, Legal);
1661   setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i16, MVT::v4i8, Legal);
1662   setLoadExtAction(ISD::SEXTLOAD, MVT::v4i16, MVT::v4i8, Legal);
1663 
1664   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8,  Legal);
1665   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Legal);
1666   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Legal);
1667 
1668   // Types natively supported:
1669   for (MVT NativeVT : {MVT::v8i1, MVT::v4i1, MVT::v2i1, MVT::v4i8,
1670                        MVT::v8i8, MVT::v2i16, MVT::v4i16, MVT::v2i32}) {
1671     setOperationAction(ISD::BUILD_VECTOR,       NativeVT, Custom);
1672     setOperationAction(ISD::EXTRACT_VECTOR_ELT, NativeVT, Custom);
1673     setOperationAction(ISD::INSERT_VECTOR_ELT,  NativeVT, Custom);
1674     setOperationAction(ISD::EXTRACT_SUBVECTOR,  NativeVT, Custom);
1675     setOperationAction(ISD::INSERT_SUBVECTOR,   NativeVT, Custom);
1676     setOperationAction(ISD::CONCAT_VECTORS,     NativeVT, Custom);
1677 
1678     setOperationAction(ISD::ADD, NativeVT, Legal);
1679     setOperationAction(ISD::SUB, NativeVT, Legal);
1680     setOperationAction(ISD::MUL, NativeVT, Legal);
1681     setOperationAction(ISD::AND, NativeVT, Legal);
1682     setOperationAction(ISD::OR,  NativeVT, Legal);
1683     setOperationAction(ISD::XOR, NativeVT, Legal);
1684 
1685     if (NativeVT.getVectorElementType() != MVT::i1)
1686       setOperationAction(ISD::SPLAT_VECTOR, NativeVT, Legal);
1687   }
1688 
1689   for (MVT VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32}) {
1690     setOperationAction(ISD::SMIN, VT, Legal);
1691     setOperationAction(ISD::SMAX, VT, Legal);
1692     setOperationAction(ISD::UMIN, VT, Legal);
1693     setOperationAction(ISD::UMAX, VT, Legal);
1694   }
1695 
1696   // Custom lower unaligned loads.
1697   // Also, for both loads and stores, verify the alignment of the address
1698   // in case it is a compile-time constant. This is a usability feature to
1699   // provide a meaningful error message to users.
1700   for (MVT VT : {MVT::i16, MVT::i32, MVT::v4i8, MVT::i64, MVT::v8i8,
1701                  MVT::v2i16, MVT::v4i16, MVT::v2i32}) {
1702     setOperationAction(ISD::LOAD,  VT, Custom);
1703     setOperationAction(ISD::STORE, VT, Custom);
1704   }
1705 
1706   for (MVT VT : {MVT::v2i16, MVT::v4i8, MVT::v8i8, MVT::v2i32, MVT::v4i16,
1707                  MVT::v2i32}) {
1708     setCondCodeAction(ISD::SETNE,  VT, Expand);
1709     setCondCodeAction(ISD::SETLE,  VT, Expand);
1710     setCondCodeAction(ISD::SETGE,  VT, Expand);
1711     setCondCodeAction(ISD::SETLT,  VT, Expand);
1712     setCondCodeAction(ISD::SETULE, VT, Expand);
1713     setCondCodeAction(ISD::SETUGE, VT, Expand);
1714     setCondCodeAction(ISD::SETULT, VT, Expand);
1715   }
1716 
1717   // Custom-lower bitcasts from i8 to v8i1.
1718   setOperationAction(ISD::BITCAST,        MVT::i8,    Custom);
1719   setOperationAction(ISD::SETCC,          MVT::v2i16, Custom);
1720   setOperationAction(ISD::VSELECT,        MVT::v4i8,  Custom);
1721   setOperationAction(ISD::VSELECT,        MVT::v2i16, Custom);
1722   setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i8,  Custom);
1723   setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom);
1724   setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i8,  Custom);
1725 
1726   // V5+.
1727   setOperationAction(ISD::FMA,  MVT::f64, Expand);
1728   setOperationAction(ISD::FADD, MVT::f64, Expand);
1729   setOperationAction(ISD::FSUB, MVT::f64, Expand);
1730   setOperationAction(ISD::FMUL, MVT::f64, Expand);
1731 
1732   setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
1733   setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
1734 
1735   setOperationAction(ISD::FP_TO_UINT, MVT::i1,  Promote);
1736   setOperationAction(ISD::FP_TO_UINT, MVT::i8,  Promote);
1737   setOperationAction(ISD::FP_TO_UINT, MVT::i16, Promote);
1738   setOperationAction(ISD::FP_TO_SINT, MVT::i1,  Promote);
1739   setOperationAction(ISD::FP_TO_SINT, MVT::i8,  Promote);
1740   setOperationAction(ISD::FP_TO_SINT, MVT::i16, Promote);
1741   setOperationAction(ISD::UINT_TO_FP, MVT::i1,  Promote);
1742   setOperationAction(ISD::UINT_TO_FP, MVT::i8,  Promote);
1743   setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote);
1744   setOperationAction(ISD::SINT_TO_FP, MVT::i1,  Promote);
1745   setOperationAction(ISD::SINT_TO_FP, MVT::i8,  Promote);
1746   setOperationAction(ISD::SINT_TO_FP, MVT::i16, Promote);
1747 
1748   // Handling of indexed loads/stores: default is "expand".
1749   //
1750   for (MVT VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64, MVT::f32, MVT::f64,
1751                  MVT::v2i16, MVT::v2i32, MVT::v4i8, MVT::v4i16, MVT::v8i8}) {
1752     setIndexedLoadAction(ISD::POST_INC, VT, Legal);
1753     setIndexedStoreAction(ISD::POST_INC, VT, Legal);
1754   }
1755 
1756   // Subtarget-specific operation actions.
1757   //
1758   if (Subtarget.hasV60Ops()) {
1759     setOperationAction(ISD::ROTL, MVT::i32, Legal);
1760     setOperationAction(ISD::ROTL, MVT::i64, Legal);
1761     setOperationAction(ISD::ROTR, MVT::i32, Legal);
1762     setOperationAction(ISD::ROTR, MVT::i64, Legal);
1763   }
1764   if (Subtarget.hasV66Ops()) {
1765     setOperationAction(ISD::FADD, MVT::f64, Legal);
1766     setOperationAction(ISD::FSUB, MVT::f64, Legal);
1767   }
1768   if (Subtarget.hasV67Ops()) {
1769     setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
1770     setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
1771     setOperationAction(ISD::FMUL,    MVT::f64, Legal);
1772   }
1773 
1774   setTargetDAGCombine(ISD::VSELECT);
1775 
1776   if (Subtarget.useHVXOps())
1777     initializeHVXLowering();
1778 
1779   computeRegisterProperties(&HRI);
1780 
1781   //
1782   // Library calls for unsupported operations
1783   //
1784   bool FastMath  = EnableFastMath;
1785 
1786   setLibcallName(RTLIB::SDIV_I32, "__hexagon_divsi3");
1787   setLibcallName(RTLIB::SDIV_I64, "__hexagon_divdi3");
1788   setLibcallName(RTLIB::UDIV_I32, "__hexagon_udivsi3");
1789   setLibcallName(RTLIB::UDIV_I64, "__hexagon_udivdi3");
1790   setLibcallName(RTLIB::SREM_I32, "__hexagon_modsi3");
1791   setLibcallName(RTLIB::SREM_I64, "__hexagon_moddi3");
1792   setLibcallName(RTLIB::UREM_I32, "__hexagon_umodsi3");
1793   setLibcallName(RTLIB::UREM_I64, "__hexagon_umoddi3");
1794 
1795   setLibcallName(RTLIB::SINTTOFP_I128_F64, "__hexagon_floattidf");
1796   setLibcallName(RTLIB::SINTTOFP_I128_F32, "__hexagon_floattisf");
1797   setLibcallName(RTLIB::FPTOUINT_F32_I128, "__hexagon_fixunssfti");
1798   setLibcallName(RTLIB::FPTOUINT_F64_I128, "__hexagon_fixunsdfti");
1799   setLibcallName(RTLIB::FPTOSINT_F32_I128, "__hexagon_fixsfti");
1800   setLibcallName(RTLIB::FPTOSINT_F64_I128, "__hexagon_fixdfti");
1801 
1802   // This is the only fast library function for sqrtd.
1803   if (FastMath)
1804     setLibcallName(RTLIB::SQRT_F64, "__hexagon_fast2_sqrtdf2");
1805 
1806   // Prefix is: nothing  for "slow-math",
1807   //            "fast2_" for V5+ fast-math double-precision
1808   // (actually, keep fast-math and fast-math2 separate for now)
1809   if (FastMath) {
1810     setLibcallName(RTLIB::ADD_F64, "__hexagon_fast_adddf3");
1811     setLibcallName(RTLIB::SUB_F64, "__hexagon_fast_subdf3");
1812     setLibcallName(RTLIB::MUL_F64, "__hexagon_fast_muldf3");
1813     setLibcallName(RTLIB::DIV_F64, "__hexagon_fast_divdf3");
1814     setLibcallName(RTLIB::DIV_F32, "__hexagon_fast_divsf3");
1815   } else {
1816     setLibcallName(RTLIB::ADD_F64, "__hexagon_adddf3");
1817     setLibcallName(RTLIB::SUB_F64, "__hexagon_subdf3");
1818     setLibcallName(RTLIB::MUL_F64, "__hexagon_muldf3");
1819     setLibcallName(RTLIB::DIV_F64, "__hexagon_divdf3");
1820     setLibcallName(RTLIB::DIV_F32, "__hexagon_divsf3");
1821   }
1822 
1823   if (FastMath)
1824     setLibcallName(RTLIB::SQRT_F32, "__hexagon_fast2_sqrtf");
1825   else
1826     setLibcallName(RTLIB::SQRT_F32, "__hexagon_sqrtf");
1827 
1828   // These cause problems when the shift amount is non-constant.
1829   setLibcallName(RTLIB::SHL_I128, nullptr);
1830   setLibcallName(RTLIB::SRL_I128, nullptr);
1831   setLibcallName(RTLIB::SRA_I128, nullptr);
1832 }
1833 
1834 const char* HexagonTargetLowering::getTargetNodeName(unsigned Opcode) const {
1835   switch ((HexagonISD::NodeType)Opcode) {
1836   case HexagonISD::ADDC:          return "HexagonISD::ADDC";
1837   case HexagonISD::SUBC:          return "HexagonISD::SUBC";
1838   case HexagonISD::ALLOCA:        return "HexagonISD::ALLOCA";
1839   case HexagonISD::AT_GOT:        return "HexagonISD::AT_GOT";
1840   case HexagonISD::AT_PCREL:      return "HexagonISD::AT_PCREL";
1841   case HexagonISD::BARRIER:       return "HexagonISD::BARRIER";
1842   case HexagonISD::CALL:          return "HexagonISD::CALL";
1843   case HexagonISD::CALLnr:        return "HexagonISD::CALLnr";
1844   case HexagonISD::CALLR:         return "HexagonISD::CALLR";
1845   case HexagonISD::COMBINE:       return "HexagonISD::COMBINE";
1846   case HexagonISD::CONST32_GP:    return "HexagonISD::CONST32_GP";
1847   case HexagonISD::CONST32:       return "HexagonISD::CONST32";
1848   case HexagonISD::CP:            return "HexagonISD::CP";
1849   case HexagonISD::DCFETCH:       return "HexagonISD::DCFETCH";
1850   case HexagonISD::EH_RETURN:     return "HexagonISD::EH_RETURN";
1851   case HexagonISD::TSTBIT:        return "HexagonISD::TSTBIT";
1852   case HexagonISD::EXTRACTU:      return "HexagonISD::EXTRACTU";
1853   case HexagonISD::INSERT:        return "HexagonISD::INSERT";
1854   case HexagonISD::JT:            return "HexagonISD::JT";
1855   case HexagonISD::RET_FLAG:      return "HexagonISD::RET_FLAG";
1856   case HexagonISD::TC_RETURN:     return "HexagonISD::TC_RETURN";
1857   case HexagonISD::VASL:          return "HexagonISD::VASL";
1858   case HexagonISD::VASR:          return "HexagonISD::VASR";
1859   case HexagonISD::VLSR:          return "HexagonISD::VLSR";
1860   case HexagonISD::VEXTRACTW:     return "HexagonISD::VEXTRACTW";
1861   case HexagonISD::VINSERTW0:     return "HexagonISD::VINSERTW0";
1862   case HexagonISD::VROR:          return "HexagonISD::VROR";
1863   case HexagonISD::READCYCLE:     return "HexagonISD::READCYCLE";
1864   case HexagonISD::PTRUE:         return "HexagonISD::PTRUE";
1865   case HexagonISD::PFALSE:        return "HexagonISD::PFALSE";
1866   case HexagonISD::D2P:           return "HexagonISD::D2P";
1867   case HexagonISD::P2D:           return "HexagonISD::P2D";
1868   case HexagonISD::V2Q:           return "HexagonISD::V2Q";
1869   case HexagonISD::Q2V:           return "HexagonISD::Q2V";
1870   case HexagonISD::QCAT:          return "HexagonISD::QCAT";
1871   case HexagonISD::QTRUE:         return "HexagonISD::QTRUE";
1872   case HexagonISD::QFALSE:        return "HexagonISD::QFALSE";
1873   case HexagonISD::TYPECAST:      return "HexagonISD::TYPECAST";
1874   case HexagonISD::VALIGN:        return "HexagonISD::VALIGN";
1875   case HexagonISD::VALIGNADDR:    return "HexagonISD::VALIGNADDR";
1876   case HexagonISD::VPACKL:        return "HexagonISD::VPACKL";
1877   case HexagonISD::VUNPACK:       return "HexagonISD::VUNPACK";
1878   case HexagonISD::VUNPACKU:      return "HexagonISD::VUNPACKU";
1879   case HexagonISD::ISEL:          return "HexagonISD::ISEL";
1880   case HexagonISD::OP_END:        break;
1881   }
1882   return nullptr;
1883 }
1884 
1885 void
1886 HexagonTargetLowering::validateConstPtrAlignment(SDValue Ptr, const SDLoc &dl,
1887       unsigned NeedAlign) const {
1888   auto *CA = dyn_cast<ConstantSDNode>(Ptr);
1889   if (!CA)
1890     return;
1891   unsigned Addr = CA->getZExtValue();
1892   unsigned HaveAlign = Addr != 0 ? 1u << countTrailingZeros(Addr) : NeedAlign;
1893   if (HaveAlign < NeedAlign) {
1894     std::string ErrMsg;
1895     raw_string_ostream O(ErrMsg);
1896     O << "Misaligned constant address: " << format_hex(Addr, 10)
1897       << " has alignment " << HaveAlign
1898       << ", but the memory access requires " << NeedAlign;
1899     if (DebugLoc DL = dl.getDebugLoc())
1900       DL.print(O << ", at ");
1901     report_fatal_error(O.str());
1902   }
1903 }
1904 
1905 // Bit-reverse Load Intrinsic: Check if the instruction is a bit reverse load
1906 // intrinsic.
1907 static bool isBrevLdIntrinsic(const Value *Inst) {
1908   unsigned ID = cast<IntrinsicInst>(Inst)->getIntrinsicID();
1909   return (ID == Intrinsic::hexagon_L2_loadrd_pbr ||
1910           ID == Intrinsic::hexagon_L2_loadri_pbr ||
1911           ID == Intrinsic::hexagon_L2_loadrh_pbr ||
1912           ID == Intrinsic::hexagon_L2_loadruh_pbr ||
1913           ID == Intrinsic::hexagon_L2_loadrb_pbr ||
1914           ID == Intrinsic::hexagon_L2_loadrub_pbr);
1915 }
1916 
1917 // Bit-reverse Load Intrinsic :Crawl up and figure out the object from previous
1918 // instruction. So far we only handle bitcast, extract value and bit reverse
1919 // load intrinsic instructions. Should we handle CGEP ?
1920 static Value *getBrevLdObject(Value *V) {
1921   if (Operator::getOpcode(V) == Instruction::ExtractValue ||
1922       Operator::getOpcode(V) == Instruction::BitCast)
1923     V = cast<Operator>(V)->getOperand(0);
1924   else if (isa<IntrinsicInst>(V) && isBrevLdIntrinsic(V))
1925     V = cast<Instruction>(V)->getOperand(0);
1926   return V;
1927 }
1928 
1929 // Bit-reverse Load Intrinsic: For a PHI Node return either an incoming edge or
1930 // a back edge. If the back edge comes from the intrinsic itself, the incoming
1931 // edge is returned.
1932 static Value *returnEdge(const PHINode *PN, Value *IntrBaseVal) {
1933   const BasicBlock *Parent = PN->getParent();
1934   int Idx = -1;
1935   for (unsigned i = 0, e = PN->getNumIncomingValues(); i < e; ++i) {
1936     BasicBlock *Blk = PN->getIncomingBlock(i);
1937     // Determine if the back edge is originated from intrinsic.
1938     if (Blk == Parent) {
1939       Value *BackEdgeVal = PN->getIncomingValue(i);
1940       Value *BaseVal;
1941       // Loop over till we return the same Value or we hit the IntrBaseVal.
1942       do {
1943         BaseVal = BackEdgeVal;
1944         BackEdgeVal = getBrevLdObject(BackEdgeVal);
1945       } while ((BaseVal != BackEdgeVal) && (IntrBaseVal != BackEdgeVal));
1946       // If the getBrevLdObject returns IntrBaseVal, we should return the
1947       // incoming edge.
1948       if (IntrBaseVal == BackEdgeVal)
1949         continue;
1950       Idx = i;
1951       break;
1952     } else // Set the node to incoming edge.
1953       Idx = i;
1954   }
1955   assert(Idx >= 0 && "Unexpected index to incoming argument in PHI");
1956   return PN->getIncomingValue(Idx);
1957 }
1958 
1959 // Bit-reverse Load Intrinsic: Figure out the underlying object the base
1960 // pointer points to, for the bit-reverse load intrinsic. Setting this to
1961 // memoperand might help alias analysis to figure out the dependencies.
1962 static Value *getUnderLyingObjectForBrevLdIntr(Value *V) {
1963   Value *IntrBaseVal = V;
1964   Value *BaseVal;
1965   // Loop over till we return the same Value, implies we either figure out
1966   // the object or we hit a PHI
1967   do {
1968     BaseVal = V;
1969     V = getBrevLdObject(V);
1970   } while (BaseVal != V);
1971 
1972   // Identify the object from PHINode.
1973   if (const PHINode *PN = dyn_cast<PHINode>(V))
1974     return returnEdge(PN, IntrBaseVal);
1975   // For non PHI nodes, the object is the last value returned by getBrevLdObject
1976   else
1977     return V;
1978 }
1979 
1980 /// Given an intrinsic, checks if on the target the intrinsic will need to map
1981 /// to a MemIntrinsicNode (touches memory). If this is the case, it returns
1982 /// true and store the intrinsic information into the IntrinsicInfo that was
1983 /// passed to the function.
1984 bool HexagonTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
1985                                                const CallInst &I,
1986                                                MachineFunction &MF,
1987                                                unsigned Intrinsic) const {
1988   switch (Intrinsic) {
1989   case Intrinsic::hexagon_L2_loadrd_pbr:
1990   case Intrinsic::hexagon_L2_loadri_pbr:
1991   case Intrinsic::hexagon_L2_loadrh_pbr:
1992   case Intrinsic::hexagon_L2_loadruh_pbr:
1993   case Intrinsic::hexagon_L2_loadrb_pbr:
1994   case Intrinsic::hexagon_L2_loadrub_pbr: {
1995     Info.opc = ISD::INTRINSIC_W_CHAIN;
1996     auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
1997     auto &Cont = I.getCalledFunction()->getParent()->getContext();
1998     // The intrinsic function call is of the form { ElTy, i8* }
1999     // @llvm.hexagon.L2.loadXX.pbr(i8*, i32). The pointer and memory access type
2000     // should be derived from ElTy.
2001     Type *ElTy = I.getCalledFunction()->getReturnType()->getStructElementType(0);
2002     Info.memVT = MVT::getVT(ElTy);
2003     llvm::Value *BasePtrVal = I.getOperand(0);
2004     Info.ptrVal = getUnderLyingObjectForBrevLdIntr(BasePtrVal);
2005     // The offset value comes through Modifier register. For now, assume the
2006     // offset is 0.
2007     Info.offset = 0;
2008     Info.align = DL.getABITypeAlign(Info.memVT.getTypeForEVT(Cont));
2009     Info.flags = MachineMemOperand::MOLoad;
2010     return true;
2011   }
2012   case Intrinsic::hexagon_V6_vgathermw:
2013   case Intrinsic::hexagon_V6_vgathermw_128B:
2014   case Intrinsic::hexagon_V6_vgathermh:
2015   case Intrinsic::hexagon_V6_vgathermh_128B:
2016   case Intrinsic::hexagon_V6_vgathermhw:
2017   case Intrinsic::hexagon_V6_vgathermhw_128B:
2018   case Intrinsic::hexagon_V6_vgathermwq:
2019   case Intrinsic::hexagon_V6_vgathermwq_128B:
2020   case Intrinsic::hexagon_V6_vgathermhq:
2021   case Intrinsic::hexagon_V6_vgathermhq_128B:
2022   case Intrinsic::hexagon_V6_vgathermhwq:
2023   case Intrinsic::hexagon_V6_vgathermhwq_128B: {
2024     const Module &M = *I.getParent()->getParent()->getParent();
2025     Info.opc = ISD::INTRINSIC_W_CHAIN;
2026     Type *VecTy = I.getArgOperand(1)->getType();
2027     Info.memVT = MVT::getVT(VecTy);
2028     Info.ptrVal = I.getArgOperand(0);
2029     Info.offset = 0;
2030     Info.align =
2031         MaybeAlign(M.getDataLayout().getTypeAllocSizeInBits(VecTy) / 8);
2032     Info.flags = MachineMemOperand::MOLoad |
2033                  MachineMemOperand::MOStore |
2034                  MachineMemOperand::MOVolatile;
2035     return true;
2036   }
2037   default:
2038     break;
2039   }
2040   return false;
2041 }
2042 
2043 bool HexagonTargetLowering::hasBitTest(SDValue X, SDValue Y) const {
2044   return X.getValueType().isScalarInteger(); // 'tstbit'
2045 }
2046 
2047 bool HexagonTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
2048   return isTruncateFree(EVT::getEVT(Ty1), EVT::getEVT(Ty2));
2049 }
2050 
2051 bool HexagonTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
2052   if (!VT1.isSimple() || !VT2.isSimple())
2053     return false;
2054   return VT1.getSimpleVT() == MVT::i64 && VT2.getSimpleVT() == MVT::i32;
2055 }
2056 
2057 bool HexagonTargetLowering::isFMAFasterThanFMulAndFAdd(
2058     const MachineFunction &MF, EVT VT) const {
2059   return isOperationLegalOrCustom(ISD::FMA, VT);
2060 }
2061 
2062 // Should we expand the build vector with shuffles?
2063 bool HexagonTargetLowering::shouldExpandBuildVectorWithShuffles(EVT VT,
2064       unsigned DefinedValues) const {
2065   return false;
2066 }
2067 
2068 bool HexagonTargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask,
2069                                                EVT VT) const {
2070   return true;
2071 }
2072 
2073 TargetLoweringBase::LegalizeTypeAction
2074 HexagonTargetLowering::getPreferredVectorAction(MVT VT) const {
2075   unsigned VecLen = VT.getVectorNumElements();
2076   MVT ElemTy = VT.getVectorElementType();
2077 
2078   if (VecLen == 1 || VT.isScalableVector())
2079     return TargetLoweringBase::TypeScalarizeVector;
2080 
2081   if (Subtarget.useHVXOps()) {
2082     unsigned Action = getPreferredHvxVectorAction(VT);
2083     if (Action != ~0u)
2084       return static_cast<TargetLoweringBase::LegalizeTypeAction>(Action);
2085   }
2086 
2087   // Always widen (remaining) vectors of i1.
2088   if (ElemTy == MVT::i1)
2089     return TargetLoweringBase::TypeWidenVector;
2090 
2091   return TargetLoweringBase::TypeSplitVector;
2092 }
2093 
2094 std::pair<SDValue, int>
2095 HexagonTargetLowering::getBaseAndOffset(SDValue Addr) const {
2096   if (Addr.getOpcode() == ISD::ADD) {
2097     SDValue Op1 = Addr.getOperand(1);
2098     if (auto *CN = dyn_cast<const ConstantSDNode>(Op1.getNode()))
2099       return { Addr.getOperand(0), CN->getSExtValue() };
2100   }
2101   return { Addr, 0 };
2102 }
2103 
2104 // Lower a vector shuffle (V1, V2, V3).  V1 and V2 are the two vectors
2105 // to select data from, V3 is the permutation.
2106 SDValue
2107 HexagonTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG)
2108       const {
2109   const auto *SVN = cast<ShuffleVectorSDNode>(Op);
2110   ArrayRef<int> AM = SVN->getMask();
2111   assert(AM.size() <= 8 && "Unexpected shuffle mask");
2112   unsigned VecLen = AM.size();
2113 
2114   MVT VecTy = ty(Op);
2115   assert(!Subtarget.isHVXVectorType(VecTy, true) &&
2116          "HVX shuffles should be legal");
2117   assert(VecTy.getSizeInBits() <= 64 && "Unexpected vector length");
2118 
2119   SDValue Op0 = Op.getOperand(0);
2120   SDValue Op1 = Op.getOperand(1);
2121   const SDLoc &dl(Op);
2122 
2123   // If the inputs are not the same as the output, bail. This is not an
2124   // error situation, but complicates the handling and the default expansion
2125   // (into BUILD_VECTOR) should be adequate.
2126   if (ty(Op0) != VecTy || ty(Op1) != VecTy)
2127     return SDValue();
2128 
2129   // Normalize the mask so that the first non-negative index comes from
2130   // the first operand.
2131   SmallVector<int,8> Mask(AM.begin(), AM.end());
2132   unsigned F = llvm::find_if(AM, [](int M) { return M >= 0; }) - AM.data();
2133   if (F == AM.size())
2134     return DAG.getUNDEF(VecTy);
2135   if (AM[F] >= int(VecLen)) {
2136     ShuffleVectorSDNode::commuteMask(Mask);
2137     std::swap(Op0, Op1);
2138   }
2139 
2140   // Express the shuffle mask in terms of bytes.
2141   SmallVector<int,8> ByteMask;
2142   unsigned ElemBytes = VecTy.getVectorElementType().getSizeInBits() / 8;
2143   for (unsigned i = 0, e = Mask.size(); i != e; ++i) {
2144     int M = Mask[i];
2145     if (M < 0) {
2146       for (unsigned j = 0; j != ElemBytes; ++j)
2147         ByteMask.push_back(-1);
2148     } else {
2149       for (unsigned j = 0; j != ElemBytes; ++j)
2150         ByteMask.push_back(M*ElemBytes + j);
2151     }
2152   }
2153   assert(ByteMask.size() <= 8);
2154 
2155   // All non-undef (non-negative) indexes are well within [0..127], so they
2156   // fit in a single byte. Build two 64-bit words:
2157   // - MaskIdx where each byte is the corresponding index (for non-negative
2158   //   indexes), and 0xFF for negative indexes, and
2159   // - MaskUnd that has 0xFF for each negative index.
2160   uint64_t MaskIdx = 0;
2161   uint64_t MaskUnd = 0;
2162   for (unsigned i = 0, e = ByteMask.size(); i != e; ++i) {
2163     unsigned S = 8*i;
2164     uint64_t M = ByteMask[i] & 0xFF;
2165     if (M == 0xFF)
2166       MaskUnd |= M << S;
2167     MaskIdx |= M << S;
2168   }
2169 
2170   if (ByteMask.size() == 4) {
2171     // Identity.
2172     if (MaskIdx == (0x03020100 | MaskUnd))
2173       return Op0;
2174     // Byte swap.
2175     if (MaskIdx == (0x00010203 | MaskUnd)) {
2176       SDValue T0 = DAG.getBitcast(MVT::i32, Op0);
2177       SDValue T1 = DAG.getNode(ISD::BSWAP, dl, MVT::i32, T0);
2178       return DAG.getBitcast(VecTy, T1);
2179     }
2180 
2181     // Byte packs.
2182     SDValue Concat10 = DAG.getNode(HexagonISD::COMBINE, dl,
2183                                    typeJoin({ty(Op1), ty(Op0)}), {Op1, Op0});
2184     if (MaskIdx == (0x06040200 | MaskUnd))
2185       return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat10}, DAG);
2186     if (MaskIdx == (0x07050301 | MaskUnd))
2187       return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat10}, DAG);
2188 
2189     SDValue Concat01 = DAG.getNode(HexagonISD::COMBINE, dl,
2190                                    typeJoin({ty(Op0), ty(Op1)}), {Op0, Op1});
2191     if (MaskIdx == (0x02000604 | MaskUnd))
2192       return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat01}, DAG);
2193     if (MaskIdx == (0x03010705 | MaskUnd))
2194       return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat01}, DAG);
2195   }
2196 
2197   if (ByteMask.size() == 8) {
2198     // Identity.
2199     if (MaskIdx == (0x0706050403020100ull | MaskUnd))
2200       return Op0;
2201     // Byte swap.
2202     if (MaskIdx == (0x0001020304050607ull | MaskUnd)) {
2203       SDValue T0 = DAG.getBitcast(MVT::i64, Op0);
2204       SDValue T1 = DAG.getNode(ISD::BSWAP, dl, MVT::i64, T0);
2205       return DAG.getBitcast(VecTy, T1);
2206     }
2207 
2208     // Halfword picks.
2209     if (MaskIdx == (0x0d0c050409080100ull | MaskUnd))
2210       return getInstr(Hexagon::S2_shuffeh, dl, VecTy, {Op1, Op0}, DAG);
2211     if (MaskIdx == (0x0f0e07060b0a0302ull | MaskUnd))
2212       return getInstr(Hexagon::S2_shuffoh, dl, VecTy, {Op1, Op0}, DAG);
2213     if (MaskIdx == (0x0d0c090805040100ull | MaskUnd))
2214       return getInstr(Hexagon::S2_vtrunewh, dl, VecTy, {Op1, Op0}, DAG);
2215     if (MaskIdx == (0x0f0e0b0a07060302ull | MaskUnd))
2216       return getInstr(Hexagon::S2_vtrunowh, dl, VecTy, {Op1, Op0}, DAG);
2217     if (MaskIdx == (0x0706030205040100ull | MaskUnd)) {
2218       VectorPair P = opSplit(Op0, dl, DAG);
2219       return getInstr(Hexagon::S2_packhl, dl, VecTy, {P.second, P.first}, DAG);
2220     }
2221 
2222     // Byte packs.
2223     if (MaskIdx == (0x0e060c040a020800ull | MaskUnd))
2224       return getInstr(Hexagon::S2_shuffeb, dl, VecTy, {Op1, Op0}, DAG);
2225     if (MaskIdx == (0x0f070d050b030901ull | MaskUnd))
2226       return getInstr(Hexagon::S2_shuffob, dl, VecTy, {Op1, Op0}, DAG);
2227   }
2228 
2229   return SDValue();
2230 }
2231 
2232 // Create a Hexagon-specific node for shifting a vector by an integer.
2233 SDValue
2234 HexagonTargetLowering::getVectorShiftByInt(SDValue Op, SelectionDAG &DAG)
2235       const {
2236   unsigned NewOpc;
2237   switch (Op.getOpcode()) {
2238     case ISD::SHL:
2239       NewOpc = HexagonISD::VASL;
2240       break;
2241     case ISD::SRA:
2242       NewOpc = HexagonISD::VASR;
2243       break;
2244     case ISD::SRL:
2245       NewOpc = HexagonISD::VLSR;
2246       break;
2247     default:
2248       llvm_unreachable("Unexpected shift opcode");
2249   }
2250 
2251   SDValue Op0 = Op.getOperand(0);
2252   SDValue Op1 = Op.getOperand(1);
2253   const SDLoc &dl(Op);
2254 
2255   switch (Op1.getOpcode()) {
2256     case ISD::BUILD_VECTOR:
2257       if (SDValue S = cast<BuildVectorSDNode>(Op1)->getSplatValue())
2258         return DAG.getNode(NewOpc, dl, ty(Op), Op0, S);
2259       break;
2260     case ISD::SPLAT_VECTOR:
2261       return DAG.getNode(NewOpc, dl, ty(Op), Op0, Op1.getOperand(0));
2262   }
2263   return SDValue();
2264 }
2265 
2266 SDValue
2267 HexagonTargetLowering::LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const {
2268   return getVectorShiftByInt(Op, DAG);
2269 }
2270 
2271 SDValue
2272 HexagonTargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const {
2273   if (isa<ConstantSDNode>(Op.getOperand(1).getNode()))
2274     return Op;
2275   return SDValue();
2276 }
2277 
2278 SDValue
2279 HexagonTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
2280   MVT ResTy = ty(Op);
2281   SDValue InpV = Op.getOperand(0);
2282   MVT InpTy = ty(InpV);
2283   assert(ResTy.getSizeInBits() == InpTy.getSizeInBits());
2284   const SDLoc &dl(Op);
2285 
2286   // Handle conversion from i8 to v8i1.
2287   if (InpTy == MVT::i8) {
2288     if (ResTy == MVT::v8i1) {
2289       SDValue Sc = DAG.getBitcast(tyScalar(InpTy), InpV);
2290       SDValue Ext = DAG.getZExtOrTrunc(Sc, dl, MVT::i32);
2291       return getInstr(Hexagon::C2_tfrrp, dl, ResTy, Ext, DAG);
2292     }
2293     return SDValue();
2294   }
2295 
2296   return Op;
2297 }
2298 
2299 bool
2300 HexagonTargetLowering::getBuildVectorConstInts(ArrayRef<SDValue> Values,
2301       MVT VecTy, SelectionDAG &DAG,
2302       MutableArrayRef<ConstantInt*> Consts) const {
2303   MVT ElemTy = VecTy.getVectorElementType();
2304   unsigned ElemWidth = ElemTy.getSizeInBits();
2305   IntegerType *IntTy = IntegerType::get(*DAG.getContext(), ElemWidth);
2306   bool AllConst = true;
2307 
2308   for (unsigned i = 0, e = Values.size(); i != e; ++i) {
2309     SDValue V = Values[i];
2310     if (V.isUndef()) {
2311       Consts[i] = ConstantInt::get(IntTy, 0);
2312       continue;
2313     }
2314     // Make sure to always cast to IntTy.
2315     if (auto *CN = dyn_cast<ConstantSDNode>(V.getNode())) {
2316       const ConstantInt *CI = CN->getConstantIntValue();
2317       Consts[i] = ConstantInt::get(IntTy, CI->getValue().getSExtValue());
2318     } else if (auto *CN = dyn_cast<ConstantFPSDNode>(V.getNode())) {
2319       const ConstantFP *CF = CN->getConstantFPValue();
2320       APInt A = CF->getValueAPF().bitcastToAPInt();
2321       Consts[i] = ConstantInt::get(IntTy, A.getZExtValue());
2322     } else {
2323       AllConst = false;
2324     }
2325   }
2326   return AllConst;
2327 }
2328 
2329 SDValue
2330 HexagonTargetLowering::buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl,
2331                                      MVT VecTy, SelectionDAG &DAG) const {
2332   MVT ElemTy = VecTy.getVectorElementType();
2333   assert(VecTy.getVectorNumElements() == Elem.size());
2334 
2335   SmallVector<ConstantInt*,4> Consts(Elem.size());
2336   bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts);
2337 
2338   unsigned First, Num = Elem.size();
2339   for (First = 0; First != Num; ++First) {
2340     if (!isUndef(Elem[First]))
2341       break;
2342   }
2343   if (First == Num)
2344     return DAG.getUNDEF(VecTy);
2345 
2346   if (AllConst &&
2347       llvm::all_of(Consts, [](ConstantInt *CI) { return CI->isZero(); }))
2348     return getZero(dl, VecTy, DAG);
2349 
2350   if (ElemTy == MVT::i16) {
2351     assert(Elem.size() == 2);
2352     if (AllConst) {
2353       uint32_t V = (Consts[0]->getZExtValue() & 0xFFFF) |
2354                    Consts[1]->getZExtValue() << 16;
2355       return DAG.getBitcast(MVT::v2i16, DAG.getConstant(V, dl, MVT::i32));
2356     }
2357     SDValue N = getInstr(Hexagon::A2_combine_ll, dl, MVT::i32,
2358                          {Elem[1], Elem[0]}, DAG);
2359     return DAG.getBitcast(MVT::v2i16, N);
2360   }
2361 
2362   if (ElemTy == MVT::i8) {
2363     // First try generating a constant.
2364     if (AllConst) {
2365       int32_t V = (Consts[0]->getZExtValue() & 0xFF) |
2366                   (Consts[1]->getZExtValue() & 0xFF) << 8 |
2367                   (Consts[1]->getZExtValue() & 0xFF) << 16 |
2368                   Consts[2]->getZExtValue() << 24;
2369       return DAG.getBitcast(MVT::v4i8, DAG.getConstant(V, dl, MVT::i32));
2370     }
2371 
2372     // Then try splat.
2373     bool IsSplat = true;
2374     for (unsigned i = First+1; i != Num; ++i) {
2375       if (Elem[i] == Elem[First] || isUndef(Elem[i]))
2376         continue;
2377       IsSplat = false;
2378       break;
2379     }
2380     if (IsSplat) {
2381       // Legalize the operand of SPLAT_VECTOR.
2382       SDValue Ext = DAG.getZExtOrTrunc(Elem[First], dl, MVT::i32);
2383       return DAG.getNode(ISD::SPLAT_VECTOR, dl, VecTy, Ext);
2384     }
2385 
2386     // Generate
2387     //   (zxtb(Elem[0]) | (zxtb(Elem[1]) << 8)) |
2388     //   (zxtb(Elem[2]) | (zxtb(Elem[3]) << 8)) << 16
2389     assert(Elem.size() == 4);
2390     SDValue Vs[4];
2391     for (unsigned i = 0; i != 4; ++i) {
2392       Vs[i] = DAG.getZExtOrTrunc(Elem[i], dl, MVT::i32);
2393       Vs[i] = DAG.getZeroExtendInReg(Vs[i], dl, MVT::i8);
2394     }
2395     SDValue S8 = DAG.getConstant(8, dl, MVT::i32);
2396     SDValue T0 = DAG.getNode(ISD::SHL, dl, MVT::i32, {Vs[1], S8});
2397     SDValue T1 = DAG.getNode(ISD::SHL, dl, MVT::i32, {Vs[3], S8});
2398     SDValue B0 = DAG.getNode(ISD::OR, dl, MVT::i32, {Vs[0], T0});
2399     SDValue B1 = DAG.getNode(ISD::OR, dl, MVT::i32, {Vs[2], T1});
2400 
2401     SDValue R = getInstr(Hexagon::A2_combine_ll, dl, MVT::i32, {B1, B0}, DAG);
2402     return DAG.getBitcast(MVT::v4i8, R);
2403   }
2404 
2405 #ifndef NDEBUG
2406   dbgs() << "VecTy: " << EVT(VecTy).getEVTString() << '\n';
2407 #endif
2408   llvm_unreachable("Unexpected vector element type");
2409 }
2410 
2411 SDValue
2412 HexagonTargetLowering::buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl,
2413                                      MVT VecTy, SelectionDAG &DAG) const {
2414   MVT ElemTy = VecTy.getVectorElementType();
2415   assert(VecTy.getVectorNumElements() == Elem.size());
2416 
2417   SmallVector<ConstantInt*,8> Consts(Elem.size());
2418   bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts);
2419 
2420   unsigned First, Num = Elem.size();
2421   for (First = 0; First != Num; ++First) {
2422     if (!isUndef(Elem[First]))
2423       break;
2424   }
2425   if (First == Num)
2426     return DAG.getUNDEF(VecTy);
2427 
2428   if (AllConst &&
2429       llvm::all_of(Consts, [](ConstantInt *CI) { return CI->isZero(); }))
2430     return getZero(dl, VecTy, DAG);
2431 
2432   // First try splat if possible.
2433   if (ElemTy == MVT::i16) {
2434     bool IsSplat = true;
2435     for (unsigned i = First+1; i != Num; ++i) {
2436       if (Elem[i] == Elem[First] || isUndef(Elem[i]))
2437         continue;
2438       IsSplat = false;
2439       break;
2440     }
2441     if (IsSplat) {
2442       // Legalize the operand of SPLAT_VECTOR
2443       SDValue Ext = DAG.getZExtOrTrunc(Elem[First], dl, MVT::i32);
2444       return DAG.getNode(ISD::SPLAT_VECTOR, dl, VecTy, Ext);
2445     }
2446   }
2447 
2448   // Then try constant.
2449   if (AllConst) {
2450     uint64_t Val = 0;
2451     unsigned W = ElemTy.getSizeInBits();
2452     uint64_t Mask = (ElemTy == MVT::i8)  ? 0xFFull
2453                   : (ElemTy == MVT::i16) ? 0xFFFFull : 0xFFFFFFFFull;
2454     for (unsigned i = 0; i != Num; ++i)
2455       Val = (Val << W) | (Consts[Num-1-i]->getZExtValue() & Mask);
2456     SDValue V0 = DAG.getConstant(Val, dl, MVT::i64);
2457     return DAG.getBitcast(VecTy, V0);
2458   }
2459 
2460   // Build two 32-bit vectors and concatenate.
2461   MVT HalfTy = MVT::getVectorVT(ElemTy, Num/2);
2462   SDValue L = (ElemTy == MVT::i32)
2463                 ? Elem[0]
2464                 : buildVector32(Elem.take_front(Num/2), dl, HalfTy, DAG);
2465   SDValue H = (ElemTy == MVT::i32)
2466                 ? Elem[1]
2467                 : buildVector32(Elem.drop_front(Num/2), dl, HalfTy, DAG);
2468   return DAG.getNode(HexagonISD::COMBINE, dl, VecTy, {H, L});
2469 }
2470 
2471 SDValue
2472 HexagonTargetLowering::extractVector(SDValue VecV, SDValue IdxV,
2473                                      const SDLoc &dl, MVT ValTy, MVT ResTy,
2474                                      SelectionDAG &DAG) const {
2475   MVT VecTy = ty(VecV);
2476   assert(!ValTy.isVector() ||
2477          VecTy.getVectorElementType() == ValTy.getVectorElementType());
2478   unsigned VecWidth = VecTy.getSizeInBits();
2479   unsigned ValWidth = ValTy.getSizeInBits();
2480   unsigned ElemWidth = VecTy.getVectorElementType().getSizeInBits();
2481   assert((VecWidth % ElemWidth) == 0);
2482   auto *IdxN = dyn_cast<ConstantSDNode>(IdxV);
2483 
2484   // Special case for v{8,4,2}i1 (the only boolean vectors legal in Hexagon
2485   // without any coprocessors).
2486   if (ElemWidth == 1) {
2487     assert(VecWidth == VecTy.getVectorNumElements() && "Sanity failure");
2488     assert(VecWidth == 8 || VecWidth == 4 || VecWidth == 2);
2489     // Check if this is an extract of the lowest bit.
2490     if (IdxN) {
2491       // Extracting the lowest bit is a no-op, but it changes the type,
2492       // so it must be kept as an operation to avoid errors related to
2493       // type mismatches.
2494       if (IdxN->isNullValue() && ValTy.getSizeInBits() == 1)
2495         return DAG.getNode(HexagonISD::TYPECAST, dl, MVT::i1, VecV);
2496     }
2497 
2498     // If the value extracted is a single bit, use tstbit.
2499     if (ValWidth == 1) {
2500       SDValue A0 = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32, {VecV}, DAG);
2501       SDValue M0 = DAG.getConstant(8 / VecWidth, dl, MVT::i32);
2502       SDValue I0 = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, M0);
2503       return DAG.getNode(HexagonISD::TSTBIT, dl, MVT::i1, A0, I0);
2504     }
2505 
2506     // Each bool vector (v2i1, v4i1, v8i1) always occupies 8 bits in
2507     // a predicate register. The elements of the vector are repeated
2508     // in the register (if necessary) so that the total number is 8.
2509     // The extracted subvector will need to be expanded in such a way.
2510     unsigned Scale = VecWidth / ValWidth;
2511 
2512     // Generate (p2d VecV) >> 8*Idx to move the interesting bytes to
2513     // position 0.
2514     assert(ty(IdxV) == MVT::i32);
2515     unsigned VecRep = 8 / VecWidth;
2516     SDValue S0 = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
2517                              DAG.getConstant(8*VecRep, dl, MVT::i32));
2518     SDValue T0 = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, VecV);
2519     SDValue T1 = DAG.getNode(ISD::SRL, dl, MVT::i64, T0, S0);
2520     while (Scale > 1) {
2521       // The longest possible subvector is at most 32 bits, so it is always
2522       // contained in the low subregister.
2523       T1 = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, T1);
2524       T1 = expandPredicate(T1, dl, DAG);
2525       Scale /= 2;
2526     }
2527 
2528     return DAG.getNode(HexagonISD::D2P, dl, ResTy, T1);
2529   }
2530 
2531   assert(VecWidth == 32 || VecWidth == 64);
2532 
2533   // Cast everything to scalar integer types.
2534   MVT ScalarTy = tyScalar(VecTy);
2535   VecV = DAG.getBitcast(ScalarTy, VecV);
2536 
2537   SDValue WidthV = DAG.getConstant(ValWidth, dl, MVT::i32);
2538   SDValue ExtV;
2539 
2540   if (IdxN) {
2541     unsigned Off = IdxN->getZExtValue() * ElemWidth;
2542     if (VecWidth == 64 && ValWidth == 32) {
2543       assert(Off == 0 || Off == 32);
2544       unsigned SubIdx = Off == 0 ? Hexagon::isub_lo : Hexagon::isub_hi;
2545       ExtV = DAG.getTargetExtractSubreg(SubIdx, dl, MVT::i32, VecV);
2546     } else if (Off == 0 && (ValWidth % 8) == 0) {
2547       ExtV = DAG.getZeroExtendInReg(VecV, dl, tyScalar(ValTy));
2548     } else {
2549       SDValue OffV = DAG.getConstant(Off, dl, MVT::i32);
2550       // The return type of EXTRACTU must be the same as the type of the
2551       // input vector.
2552       ExtV = DAG.getNode(HexagonISD::EXTRACTU, dl, ScalarTy,
2553                          {VecV, WidthV, OffV});
2554     }
2555   } else {
2556     if (ty(IdxV) != MVT::i32)
2557       IdxV = DAG.getZExtOrTrunc(IdxV, dl, MVT::i32);
2558     SDValue OffV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
2559                                DAG.getConstant(ElemWidth, dl, MVT::i32));
2560     ExtV = DAG.getNode(HexagonISD::EXTRACTU, dl, ScalarTy,
2561                        {VecV, WidthV, OffV});
2562   }
2563 
2564   // Cast ExtV to the requested result type.
2565   ExtV = DAG.getZExtOrTrunc(ExtV, dl, tyScalar(ResTy));
2566   ExtV = DAG.getBitcast(ResTy, ExtV);
2567   return ExtV;
2568 }
2569 
2570 SDValue
2571 HexagonTargetLowering::insertVector(SDValue VecV, SDValue ValV, SDValue IdxV,
2572                                     const SDLoc &dl, MVT ValTy,
2573                                     SelectionDAG &DAG) const {
2574   MVT VecTy = ty(VecV);
2575   if (VecTy.getVectorElementType() == MVT::i1) {
2576     MVT ValTy = ty(ValV);
2577     assert(ValTy.getVectorElementType() == MVT::i1);
2578     SDValue ValR = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, ValV);
2579     unsigned VecLen = VecTy.getVectorNumElements();
2580     unsigned Scale = VecLen / ValTy.getVectorNumElements();
2581     assert(Scale > 1);
2582 
2583     for (unsigned R = Scale; R > 1; R /= 2) {
2584       ValR = contractPredicate(ValR, dl, DAG);
2585       ValR = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64,
2586                          DAG.getUNDEF(MVT::i32), ValR);
2587     }
2588     // The longest possible subvector is at most 32 bits, so it is always
2589     // contained in the low subregister.
2590     ValR = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, ValR);
2591 
2592     unsigned ValBytes = 64 / Scale;
2593     SDValue Width = DAG.getConstant(ValBytes*8, dl, MVT::i32);
2594     SDValue Idx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
2595                               DAG.getConstant(8, dl, MVT::i32));
2596     SDValue VecR = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, VecV);
2597     SDValue Ins = DAG.getNode(HexagonISD::INSERT, dl, MVT::i32,
2598                               {VecR, ValR, Width, Idx});
2599     return DAG.getNode(HexagonISD::D2P, dl, VecTy, Ins);
2600   }
2601 
2602   unsigned VecWidth = VecTy.getSizeInBits();
2603   unsigned ValWidth = ValTy.getSizeInBits();
2604   assert(VecWidth == 32 || VecWidth == 64);
2605   assert((VecWidth % ValWidth) == 0);
2606 
2607   // Cast everything to scalar integer types.
2608   MVT ScalarTy = MVT::getIntegerVT(VecWidth);
2609   // The actual type of ValV may be different than ValTy (which is related
2610   // to the vector type).
2611   unsigned VW = ty(ValV).getSizeInBits();
2612   ValV = DAG.getBitcast(MVT::getIntegerVT(VW), ValV);
2613   VecV = DAG.getBitcast(ScalarTy, VecV);
2614   if (VW != VecWidth)
2615     ValV = DAG.getAnyExtOrTrunc(ValV, dl, ScalarTy);
2616 
2617   SDValue WidthV = DAG.getConstant(ValWidth, dl, MVT::i32);
2618   SDValue InsV;
2619 
2620   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(IdxV)) {
2621     unsigned W = C->getZExtValue() * ValWidth;
2622     SDValue OffV = DAG.getConstant(W, dl, MVT::i32);
2623     InsV = DAG.getNode(HexagonISD::INSERT, dl, ScalarTy,
2624                        {VecV, ValV, WidthV, OffV});
2625   } else {
2626     if (ty(IdxV) != MVT::i32)
2627       IdxV = DAG.getZExtOrTrunc(IdxV, dl, MVT::i32);
2628     SDValue OffV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, WidthV);
2629     InsV = DAG.getNode(HexagonISD::INSERT, dl, ScalarTy,
2630                        {VecV, ValV, WidthV, OffV});
2631   }
2632 
2633   return DAG.getNode(ISD::BITCAST, dl, VecTy, InsV);
2634 }
2635 
2636 SDValue
2637 HexagonTargetLowering::expandPredicate(SDValue Vec32, const SDLoc &dl,
2638                                        SelectionDAG &DAG) const {
2639   assert(ty(Vec32).getSizeInBits() == 32);
2640   if (isUndef(Vec32))
2641     return DAG.getUNDEF(MVT::i64);
2642   return getInstr(Hexagon::S2_vsxtbh, dl, MVT::i64, {Vec32}, DAG);
2643 }
2644 
2645 SDValue
2646 HexagonTargetLowering::contractPredicate(SDValue Vec64, const SDLoc &dl,
2647                                          SelectionDAG &DAG) const {
2648   assert(ty(Vec64).getSizeInBits() == 64);
2649   if (isUndef(Vec64))
2650     return DAG.getUNDEF(MVT::i32);
2651   return getInstr(Hexagon::S2_vtrunehb, dl, MVT::i32, {Vec64}, DAG);
2652 }
2653 
2654 SDValue
2655 HexagonTargetLowering::getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG)
2656       const {
2657   if (Ty.isVector()) {
2658     assert(Ty.isInteger() && "Only integer vectors are supported here");
2659     unsigned W = Ty.getSizeInBits();
2660     if (W <= 64)
2661       return DAG.getBitcast(Ty, DAG.getConstant(0, dl, MVT::getIntegerVT(W)));
2662     return DAG.getNode(ISD::SPLAT_VECTOR, dl, Ty, getZero(dl, MVT::i32, DAG));
2663   }
2664 
2665   if (Ty.isInteger())
2666     return DAG.getConstant(0, dl, Ty);
2667   if (Ty.isFloatingPoint())
2668     return DAG.getConstantFP(0.0, dl, Ty);
2669   llvm_unreachable("Invalid type for zero");
2670 }
2671 
2672 SDValue
2673 HexagonTargetLowering::appendUndef(SDValue Val, MVT ResTy, SelectionDAG &DAG)
2674       const {
2675   MVT ValTy = ty(Val);
2676   assert(ValTy.getVectorElementType() == ResTy.getVectorElementType());
2677 
2678   unsigned ValLen = ValTy.getVectorNumElements();
2679   unsigned ResLen = ResTy.getVectorNumElements();
2680   if (ValLen == ResLen)
2681     return Val;
2682 
2683   const SDLoc &dl(Val);
2684   assert(ValLen < ResLen);
2685   assert(ResLen % ValLen == 0);
2686 
2687   SmallVector<SDValue, 4> Concats = {Val};
2688   for (unsigned i = 1, e = ResLen / ValLen; i < e; ++i)
2689     Concats.push_back(DAG.getUNDEF(ValTy));
2690 
2691   return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy, Concats);
2692 }
2693 
2694 SDValue
2695 HexagonTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
2696   MVT VecTy = ty(Op);
2697   unsigned BW = VecTy.getSizeInBits();
2698   const SDLoc &dl(Op);
2699   SmallVector<SDValue,8> Ops;
2700   for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i)
2701     Ops.push_back(Op.getOperand(i));
2702 
2703   if (BW == 32)
2704     return buildVector32(Ops, dl, VecTy, DAG);
2705   if (BW == 64)
2706     return buildVector64(Ops, dl, VecTy, DAG);
2707 
2708   if (VecTy == MVT::v8i1 || VecTy == MVT::v4i1 || VecTy == MVT::v2i1) {
2709     // Check if this is a special case or all-0 or all-1.
2710     bool All0 = true, All1 = true;
2711     for (SDValue P : Ops) {
2712       auto *CN = dyn_cast<ConstantSDNode>(P.getNode());
2713       if (CN == nullptr) {
2714         All0 = All1 = false;
2715         break;
2716       }
2717       uint32_t C = CN->getZExtValue();
2718       All0 &= (C == 0);
2719       All1 &= (C == 1);
2720     }
2721     if (All0)
2722       return DAG.getNode(HexagonISD::PFALSE, dl, VecTy);
2723     if (All1)
2724       return DAG.getNode(HexagonISD::PTRUE, dl, VecTy);
2725 
2726     // For each i1 element in the resulting predicate register, put 1
2727     // shifted by the index of the element into a general-purpose register,
2728     // then or them together and transfer it back into a predicate register.
2729     SDValue Rs[8];
2730     SDValue Z = getZero(dl, MVT::i32, DAG);
2731     // Always produce 8 bits, repeat inputs if necessary.
2732     unsigned Rep = 8 / VecTy.getVectorNumElements();
2733     for (unsigned i = 0; i != 8; ++i) {
2734       SDValue S = DAG.getConstant(1ull << i, dl, MVT::i32);
2735       Rs[i] = DAG.getSelect(dl, MVT::i32, Ops[i/Rep], S, Z);
2736     }
2737     for (ArrayRef<SDValue> A(Rs); A.size() != 1; A = A.drop_back(A.size()/2)) {
2738       for (unsigned i = 0, e = A.size()/2; i != e; ++i)
2739         Rs[i] = DAG.getNode(ISD::OR, dl, MVT::i32, Rs[2*i], Rs[2*i+1]);
2740     }
2741     // Move the value directly to a predicate register.
2742     return getInstr(Hexagon::C2_tfrrp, dl, VecTy, {Rs[0]}, DAG);
2743   }
2744 
2745   return SDValue();
2746 }
2747 
2748 SDValue
2749 HexagonTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
2750                                            SelectionDAG &DAG) const {
2751   MVT VecTy = ty(Op);
2752   const SDLoc &dl(Op);
2753   if (VecTy.getSizeInBits() == 64) {
2754     assert(Op.getNumOperands() == 2);
2755     return DAG.getNode(HexagonISD::COMBINE, dl, VecTy, Op.getOperand(1),
2756                        Op.getOperand(0));
2757   }
2758 
2759   MVT ElemTy = VecTy.getVectorElementType();
2760   if (ElemTy == MVT::i1) {
2761     assert(VecTy == MVT::v2i1 || VecTy == MVT::v4i1 || VecTy == MVT::v8i1);
2762     MVT OpTy = ty(Op.getOperand(0));
2763     // Scale is how many times the operands need to be contracted to match
2764     // the representation in the target register.
2765     unsigned Scale = VecTy.getVectorNumElements() / OpTy.getVectorNumElements();
2766     assert(Scale == Op.getNumOperands() && Scale > 1);
2767 
2768     // First, convert all bool vectors to integers, then generate pairwise
2769     // inserts to form values of doubled length. Up until there are only
2770     // two values left to concatenate, all of these values will fit in a
2771     // 32-bit integer, so keep them as i32 to use 32-bit inserts.
2772     SmallVector<SDValue,4> Words[2];
2773     unsigned IdxW = 0;
2774 
2775     for (SDValue P : Op.getNode()->op_values()) {
2776       SDValue W = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, P);
2777       for (unsigned R = Scale; R > 1; R /= 2) {
2778         W = contractPredicate(W, dl, DAG);
2779         W = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64,
2780                         DAG.getUNDEF(MVT::i32), W);
2781       }
2782       W = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, W);
2783       Words[IdxW].push_back(W);
2784     }
2785 
2786     while (Scale > 2) {
2787       SDValue WidthV = DAG.getConstant(64 / Scale, dl, MVT::i32);
2788       Words[IdxW ^ 1].clear();
2789 
2790       for (unsigned i = 0, e = Words[IdxW].size(); i != e; i += 2) {
2791         SDValue W0 = Words[IdxW][i], W1 = Words[IdxW][i+1];
2792         // Insert W1 into W0 right next to the significant bits of W0.
2793         SDValue T = DAG.getNode(HexagonISD::INSERT, dl, MVT::i32,
2794                                 {W0, W1, WidthV, WidthV});
2795         Words[IdxW ^ 1].push_back(T);
2796       }
2797       IdxW ^= 1;
2798       Scale /= 2;
2799     }
2800 
2801     // Another sanity check. At this point there should only be two words
2802     // left, and Scale should be 2.
2803     assert(Scale == 2 && Words[IdxW].size() == 2);
2804 
2805     SDValue WW = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64,
2806                              Words[IdxW][1], Words[IdxW][0]);
2807     return DAG.getNode(HexagonISD::D2P, dl, VecTy, WW);
2808   }
2809 
2810   return SDValue();
2811 }
2812 
2813 SDValue
2814 HexagonTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
2815                                                SelectionDAG &DAG) const {
2816   SDValue Vec = Op.getOperand(0);
2817   MVT ElemTy = ty(Vec).getVectorElementType();
2818   return extractVector(Vec, Op.getOperand(1), SDLoc(Op), ElemTy, ty(Op), DAG);
2819 }
2820 
2821 SDValue
2822 HexagonTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
2823                                               SelectionDAG &DAG) const {
2824   return extractVector(Op.getOperand(0), Op.getOperand(1), SDLoc(Op),
2825                        ty(Op), ty(Op), DAG);
2826 }
2827 
2828 SDValue
2829 HexagonTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
2830                                               SelectionDAG &DAG) const {
2831   return insertVector(Op.getOperand(0), Op.getOperand(1), Op.getOperand(2),
2832                       SDLoc(Op), ty(Op).getVectorElementType(), DAG);
2833 }
2834 
2835 SDValue
2836 HexagonTargetLowering::LowerINSERT_SUBVECTOR(SDValue Op,
2837                                              SelectionDAG &DAG) const {
2838   SDValue ValV = Op.getOperand(1);
2839   return insertVector(Op.getOperand(0), ValV, Op.getOperand(2),
2840                       SDLoc(Op), ty(ValV), DAG);
2841 }
2842 
2843 bool
2844 HexagonTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
2845   // Assuming the caller does not have either a signext or zeroext modifier, and
2846   // only one value is accepted, any reasonable truncation is allowed.
2847   if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
2848     return false;
2849 
2850   // FIXME: in principle up to 64-bit could be made safe, but it would be very
2851   // fragile at the moment: any support for multiple value returns would be
2852   // liable to disallow tail calls involving i64 -> iN truncation in many cases.
2853   return Ty1->getPrimitiveSizeInBits() <= 32;
2854 }
2855 
2856 SDValue
2857 HexagonTargetLowering::LowerLoad(SDValue Op, SelectionDAG &DAG) const {
2858   LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
2859   unsigned ClaimAlign = LN->getAlignment();
2860   validateConstPtrAlignment(LN->getBasePtr(), SDLoc(Op), ClaimAlign);
2861   // Call LowerUnalignedLoad for all loads, it recognizes loads that
2862   // don't need extra aligning.
2863   return LowerUnalignedLoad(Op, DAG);
2864 }
2865 
2866 SDValue
2867 HexagonTargetLowering::LowerStore(SDValue Op, SelectionDAG &DAG) const {
2868   StoreSDNode *SN = cast<StoreSDNode>(Op.getNode());
2869   unsigned ClaimAlign = SN->getAlignment();
2870   SDValue Ptr = SN->getBasePtr();
2871   const SDLoc &dl(Op);
2872   validateConstPtrAlignment(Ptr, dl, ClaimAlign);
2873 
2874   MVT StoreTy = SN->getMemoryVT().getSimpleVT();
2875   unsigned NeedAlign = Subtarget.getTypeAlignment(StoreTy);
2876   if (ClaimAlign < NeedAlign)
2877     return expandUnalignedStore(SN, DAG);
2878   return Op;
2879 }
2880 
2881 SDValue
2882 HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG)
2883       const {
2884   LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
2885   MVT LoadTy = ty(Op);
2886   unsigned NeedAlign = Subtarget.getTypeAlignment(LoadTy);
2887   unsigned HaveAlign = LN->getAlignment();
2888   if (HaveAlign >= NeedAlign)
2889     return Op;
2890 
2891   const SDLoc &dl(Op);
2892   const DataLayout &DL = DAG.getDataLayout();
2893   LLVMContext &Ctx = *DAG.getContext();
2894 
2895   // If the load aligning is disabled or the load can be broken up into two
2896   // smaller legal loads, do the default (target-independent) expansion.
2897   bool DoDefault = false;
2898   // Handle it in the default way if this is an indexed load.
2899   if (!LN->isUnindexed())
2900     DoDefault = true;
2901 
2902   if (!AlignLoads) {
2903     if (allowsMemoryAccessForAlignment(Ctx, DL, LN->getMemoryVT(),
2904                                        *LN->getMemOperand()))
2905       return Op;
2906     DoDefault = true;
2907   }
2908   if (!DoDefault && (2 * HaveAlign) == NeedAlign) {
2909     // The PartTy is the equivalent of "getLoadableTypeOfSize(HaveAlign)".
2910     MVT PartTy = HaveAlign <= 8 ? MVT::getIntegerVT(8 * HaveAlign)
2911                                 : MVT::getVectorVT(MVT::i8, HaveAlign);
2912     DoDefault =
2913         allowsMemoryAccessForAlignment(Ctx, DL, PartTy, *LN->getMemOperand());
2914   }
2915   if (DoDefault) {
2916     std::pair<SDValue, SDValue> P = expandUnalignedLoad(LN, DAG);
2917     return DAG.getMergeValues({P.first, P.second}, dl);
2918   }
2919 
2920   // The code below generates two loads, both aligned as NeedAlign, and
2921   // with the distance of NeedAlign between them. For that to cover the
2922   // bits that need to be loaded (and without overlapping), the size of
2923   // the loads should be equal to NeedAlign. This is true for all loadable
2924   // types, but add an assertion in case something changes in the future.
2925   assert(LoadTy.getSizeInBits() == 8*NeedAlign);
2926 
2927   unsigned LoadLen = NeedAlign;
2928   SDValue Base = LN->getBasePtr();
2929   SDValue Chain = LN->getChain();
2930   auto BO = getBaseAndOffset(Base);
2931   unsigned BaseOpc = BO.first.getOpcode();
2932   if (BaseOpc == HexagonISD::VALIGNADDR && BO.second % LoadLen == 0)
2933     return Op;
2934 
2935   if (BO.second % LoadLen != 0) {
2936     BO.first = DAG.getNode(ISD::ADD, dl, MVT::i32, BO.first,
2937                            DAG.getConstant(BO.second % LoadLen, dl, MVT::i32));
2938     BO.second -= BO.second % LoadLen;
2939   }
2940   SDValue BaseNoOff = (BaseOpc != HexagonISD::VALIGNADDR)
2941       ? DAG.getNode(HexagonISD::VALIGNADDR, dl, MVT::i32, BO.first,
2942                     DAG.getConstant(NeedAlign, dl, MVT::i32))
2943       : BO.first;
2944   SDValue Base0 =
2945       DAG.getMemBasePlusOffset(BaseNoOff, TypeSize::Fixed(BO.second), dl);
2946   SDValue Base1 = DAG.getMemBasePlusOffset(
2947       BaseNoOff, TypeSize::Fixed(BO.second + LoadLen), dl);
2948 
2949   MachineMemOperand *WideMMO = nullptr;
2950   if (MachineMemOperand *MMO = LN->getMemOperand()) {
2951     MachineFunction &MF = DAG.getMachineFunction();
2952     WideMMO = MF.getMachineMemOperand(
2953         MMO->getPointerInfo(), MMO->getFlags(), 2 * LoadLen, Align(LoadLen),
2954         MMO->getAAInfo(), MMO->getRanges(), MMO->getSyncScopeID(),
2955         MMO->getOrdering(), MMO->getFailureOrdering());
2956   }
2957 
2958   SDValue Load0 = DAG.getLoad(LoadTy, dl, Chain, Base0, WideMMO);
2959   SDValue Load1 = DAG.getLoad(LoadTy, dl, Chain, Base1, WideMMO);
2960 
2961   SDValue Aligned = DAG.getNode(HexagonISD::VALIGN, dl, LoadTy,
2962                                 {Load1, Load0, BaseNoOff.getOperand(0)});
2963   SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
2964                                  Load0.getValue(1), Load1.getValue(1));
2965   SDValue M = DAG.getMergeValues({Aligned, NewChain}, dl);
2966   return M;
2967 }
2968 
2969 SDValue
2970 HexagonTargetLowering::LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const {
2971   SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
2972   auto *CY = dyn_cast<ConstantSDNode>(Y);
2973   if (!CY)
2974     return SDValue();
2975 
2976   const SDLoc &dl(Op);
2977   SDVTList VTs = Op.getNode()->getVTList();
2978   assert(VTs.NumVTs == 2);
2979   assert(VTs.VTs[1] == MVT::i1);
2980   unsigned Opc = Op.getOpcode();
2981 
2982   if (CY) {
2983     uint32_t VY = CY->getZExtValue();
2984     assert(VY != 0 && "This should have been folded");
2985     // X +/- 1
2986     if (VY != 1)
2987       return SDValue();
2988 
2989     if (Opc == ISD::UADDO) {
2990       SDValue Op = DAG.getNode(ISD::ADD, dl, VTs.VTs[0], {X, Y});
2991       SDValue Ov = DAG.getSetCC(dl, MVT::i1, Op, getZero(dl, ty(Op), DAG),
2992                                 ISD::SETEQ);
2993       return DAG.getMergeValues({Op, Ov}, dl);
2994     }
2995     if (Opc == ISD::USUBO) {
2996       SDValue Op = DAG.getNode(ISD::SUB, dl, VTs.VTs[0], {X, Y});
2997       SDValue Ov = DAG.getSetCC(dl, MVT::i1, Op,
2998                                 DAG.getConstant(-1, dl, ty(Op)), ISD::SETEQ);
2999       return DAG.getMergeValues({Op, Ov}, dl);
3000     }
3001   }
3002 
3003   return SDValue();
3004 }
3005 
3006 SDValue
3007 HexagonTargetLowering::LowerAddSubCarry(SDValue Op, SelectionDAG &DAG) const {
3008   const SDLoc &dl(Op);
3009   unsigned Opc = Op.getOpcode();
3010   SDValue X = Op.getOperand(0), Y = Op.getOperand(1), C = Op.getOperand(2);
3011 
3012   if (Opc == ISD::ADDCARRY)
3013     return DAG.getNode(HexagonISD::ADDC, dl, Op.getNode()->getVTList(),
3014                        { X, Y, C });
3015 
3016   EVT CarryTy = C.getValueType();
3017   SDValue SubC = DAG.getNode(HexagonISD::SUBC, dl, Op.getNode()->getVTList(),
3018                              { X, Y, DAG.getLogicalNOT(dl, C, CarryTy) });
3019   SDValue Out[] = { SubC.getValue(0),
3020                     DAG.getLogicalNOT(dl, SubC.getValue(1), CarryTy) };
3021   return DAG.getMergeValues(Out, dl);
3022 }
3023 
3024 SDValue
3025 HexagonTargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
3026   SDValue Chain     = Op.getOperand(0);
3027   SDValue Offset    = Op.getOperand(1);
3028   SDValue Handler   = Op.getOperand(2);
3029   SDLoc dl(Op);
3030   auto PtrVT = getPointerTy(DAG.getDataLayout());
3031 
3032   // Mark function as containing a call to EH_RETURN.
3033   HexagonMachineFunctionInfo *FuncInfo =
3034     DAG.getMachineFunction().getInfo<HexagonMachineFunctionInfo>();
3035   FuncInfo->setHasEHReturn();
3036 
3037   unsigned OffsetReg = Hexagon::R28;
3038 
3039   SDValue StoreAddr =
3040       DAG.getNode(ISD::ADD, dl, PtrVT, DAG.getRegister(Hexagon::R30, PtrVT),
3041                   DAG.getIntPtrConstant(4, dl));
3042   Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo());
3043   Chain = DAG.getCopyToReg(Chain, dl, OffsetReg, Offset);
3044 
3045   // Not needed we already use it as explict input to EH_RETURN.
3046   // MF.getRegInfo().addLiveOut(OffsetReg);
3047 
3048   return DAG.getNode(HexagonISD::EH_RETURN, dl, MVT::Other, Chain);
3049 }
3050 
3051 SDValue
3052 HexagonTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
3053   unsigned Opc = Op.getOpcode();
3054 
3055   // Handle INLINEASM first.
3056   if (Opc == ISD::INLINEASM || Opc == ISD::INLINEASM_BR)
3057     return LowerINLINEASM(Op, DAG);
3058 
3059   if (isHvxOperation(Op.getNode(), DAG)) {
3060     // If HVX lowering returns nothing, try the default lowering.
3061     if (SDValue V = LowerHvxOperation(Op, DAG))
3062       return V;
3063   }
3064 
3065   switch (Opc) {
3066     default:
3067 #ifndef NDEBUG
3068       Op.getNode()->dumpr(&DAG);
3069       if (Opc > HexagonISD::OP_BEGIN && Opc < HexagonISD::OP_END)
3070         errs() << "Error: check for a non-legal type in this operation\n";
3071 #endif
3072       llvm_unreachable("Should not custom lower this!");
3073     case ISD::CONCAT_VECTORS:       return LowerCONCAT_VECTORS(Op, DAG);
3074     case ISD::INSERT_SUBVECTOR:     return LowerINSERT_SUBVECTOR(Op, DAG);
3075     case ISD::INSERT_VECTOR_ELT:    return LowerINSERT_VECTOR_ELT(Op, DAG);
3076     case ISD::EXTRACT_SUBVECTOR:    return LowerEXTRACT_SUBVECTOR(Op, DAG);
3077     case ISD::EXTRACT_VECTOR_ELT:   return LowerEXTRACT_VECTOR_ELT(Op, DAG);
3078     case ISD::BUILD_VECTOR:         return LowerBUILD_VECTOR(Op, DAG);
3079     case ISD::VECTOR_SHUFFLE:       return LowerVECTOR_SHUFFLE(Op, DAG);
3080     case ISD::BITCAST:              return LowerBITCAST(Op, DAG);
3081     case ISD::LOAD:                 return LowerLoad(Op, DAG);
3082     case ISD::STORE:                return LowerStore(Op, DAG);
3083     case ISD::UADDO:
3084     case ISD::USUBO:                return LowerUAddSubO(Op, DAG);
3085     case ISD::ADDCARRY:
3086     case ISD::SUBCARRY:             return LowerAddSubCarry(Op, DAG);
3087     case ISD::SRA:
3088     case ISD::SHL:
3089     case ISD::SRL:                  return LowerVECTOR_SHIFT(Op, DAG);
3090     case ISD::ROTL:                 return LowerROTL(Op, DAG);
3091     case ISD::ConstantPool:         return LowerConstantPool(Op, DAG);
3092     case ISD::JumpTable:            return LowerJumpTable(Op, DAG);
3093     case ISD::EH_RETURN:            return LowerEH_RETURN(Op, DAG);
3094     case ISD::RETURNADDR:           return LowerRETURNADDR(Op, DAG);
3095     case ISD::FRAMEADDR:            return LowerFRAMEADDR(Op, DAG);
3096     case ISD::GlobalTLSAddress:     return LowerGlobalTLSAddress(Op, DAG);
3097     case ISD::ATOMIC_FENCE:         return LowerATOMIC_FENCE(Op, DAG);
3098     case ISD::GlobalAddress:        return LowerGLOBALADDRESS(Op, DAG);
3099     case ISD::BlockAddress:         return LowerBlockAddress(Op, DAG);
3100     case ISD::GLOBAL_OFFSET_TABLE:  return LowerGLOBAL_OFFSET_TABLE(Op, DAG);
3101     case ISD::VACOPY:               return LowerVACOPY(Op, DAG);
3102     case ISD::VASTART:              return LowerVASTART(Op, DAG);
3103     case ISD::DYNAMIC_STACKALLOC:   return LowerDYNAMIC_STACKALLOC(Op, DAG);
3104     case ISD::SETCC:                return LowerSETCC(Op, DAG);
3105     case ISD::VSELECT:              return LowerVSELECT(Op, DAG);
3106     case ISD::INTRINSIC_WO_CHAIN:   return LowerINTRINSIC_WO_CHAIN(Op, DAG);
3107     case ISD::INTRINSIC_VOID:       return LowerINTRINSIC_VOID(Op, DAG);
3108     case ISD::PREFETCH:             return LowerPREFETCH(Op, DAG);
3109     case ISD::READCYCLECOUNTER:     return LowerREADCYCLECOUNTER(Op, DAG);
3110       break;
3111   }
3112 
3113   return SDValue();
3114 }
3115 
3116 void
3117 HexagonTargetLowering::LowerOperationWrapper(SDNode *N,
3118                                              SmallVectorImpl<SDValue> &Results,
3119                                              SelectionDAG &DAG) const {
3120   if (isHvxOperation(N, DAG)) {
3121     LowerHvxOperationWrapper(N, Results, DAG);
3122     if (!Results.empty())
3123       return;
3124   }
3125 
3126   // We are only custom-lowering stores to verify the alignment of the
3127   // address if it is a compile-time constant. Since a store can be modified
3128   // during type-legalization (the value being stored may need legalization),
3129   // return empty Results here to indicate that we don't really make any
3130   // changes in the custom lowering.
3131   if (N->getOpcode() != ISD::STORE)
3132     return TargetLowering::LowerOperationWrapper(N, Results, DAG);
3133 }
3134 
3135 void
3136 HexagonTargetLowering::ReplaceNodeResults(SDNode *N,
3137                                           SmallVectorImpl<SDValue> &Results,
3138                                           SelectionDAG &DAG) const {
3139   if (isHvxOperation(N, DAG)) {
3140     ReplaceHvxNodeResults(N, Results, DAG);
3141     if (!Results.empty())
3142       return;
3143   }
3144 
3145   const SDLoc &dl(N);
3146   switch (N->getOpcode()) {
3147     case ISD::SRL:
3148     case ISD::SRA:
3149     case ISD::SHL:
3150       return;
3151     case ISD::BITCAST:
3152       // Handle a bitcast from v8i1 to i8.
3153       if (N->getValueType(0) == MVT::i8) {
3154         if (N->getOperand(0).getValueType() == MVT::v8i1) {
3155           SDValue P = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32,
3156                                N->getOperand(0), DAG);
3157           SDValue T = DAG.getAnyExtOrTrunc(P, dl, MVT::i8);
3158           Results.push_back(T);
3159         }
3160       }
3161       break;
3162   }
3163 }
3164 
3165 SDValue
3166 HexagonTargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI)
3167       const {
3168   if (isHvxOperation(N, DCI.DAG)) {
3169     if (SDValue V = PerformHvxDAGCombine(N, DCI))
3170       return V;
3171     return SDValue();
3172   }
3173 
3174   if (DCI.isBeforeLegalizeOps())
3175     return SDValue();
3176 
3177   SDValue Op(N, 0);
3178   const SDLoc &dl(Op);
3179   unsigned Opc = Op.getOpcode();
3180 
3181   if (Opc == HexagonISD::P2D) {
3182     SDValue P = Op.getOperand(0);
3183     switch (P.getOpcode()) {
3184       case HexagonISD::PTRUE:
3185         return DCI.DAG.getConstant(-1, dl, ty(Op));
3186       case HexagonISD::PFALSE:
3187         return getZero(dl, ty(Op), DCI.DAG);
3188       default:
3189         break;
3190     }
3191   } else if (Opc == ISD::VSELECT) {
3192     // This is pretty much duplicated in HexagonISelLoweringHVX...
3193     //
3194     // (vselect (xor x, ptrue), v0, v1) -> (vselect x, v1, v0)
3195     SDValue Cond = Op.getOperand(0);
3196     if (Cond->getOpcode() == ISD::XOR) {
3197       SDValue C0 = Cond.getOperand(0), C1 = Cond.getOperand(1);
3198       if (C1->getOpcode() == HexagonISD::PTRUE) {
3199         SDValue VSel = DCI.DAG.getNode(ISD::VSELECT, dl, ty(Op), C0,
3200                                        Op.getOperand(2), Op.getOperand(1));
3201         return VSel;
3202       }
3203     }
3204   }
3205 
3206   return SDValue();
3207 }
3208 
3209 /// Returns relocation base for the given PIC jumptable.
3210 SDValue
3211 HexagonTargetLowering::getPICJumpTableRelocBase(SDValue Table,
3212                                                 SelectionDAG &DAG) const {
3213   int Idx = cast<JumpTableSDNode>(Table)->getIndex();
3214   EVT VT = Table.getValueType();
3215   SDValue T = DAG.getTargetJumpTable(Idx, VT, HexagonII::MO_PCREL);
3216   return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Table), VT, T);
3217 }
3218 
3219 //===----------------------------------------------------------------------===//
3220 // Inline Assembly Support
3221 //===----------------------------------------------------------------------===//
3222 
3223 TargetLowering::ConstraintType
3224 HexagonTargetLowering::getConstraintType(StringRef Constraint) const {
3225   if (Constraint.size() == 1) {
3226     switch (Constraint[0]) {
3227       case 'q':
3228       case 'v':
3229         if (Subtarget.useHVXOps())
3230           return C_RegisterClass;
3231         break;
3232       case 'a':
3233         return C_RegisterClass;
3234       default:
3235         break;
3236     }
3237   }
3238   return TargetLowering::getConstraintType(Constraint);
3239 }
3240 
3241 std::pair<unsigned, const TargetRegisterClass*>
3242 HexagonTargetLowering::getRegForInlineAsmConstraint(
3243     const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
3244 
3245   if (Constraint.size() == 1) {
3246     switch (Constraint[0]) {
3247     case 'r':   // R0-R31
3248       switch (VT.SimpleTy) {
3249       default:
3250         return {0u, nullptr};
3251       case MVT::i1:
3252       case MVT::i8:
3253       case MVT::i16:
3254       case MVT::i32:
3255       case MVT::f32:
3256         return {0u, &Hexagon::IntRegsRegClass};
3257       case MVT::i64:
3258       case MVT::f64:
3259         return {0u, &Hexagon::DoubleRegsRegClass};
3260       }
3261       break;
3262     case 'a': // M0-M1
3263       if (VT != MVT::i32)
3264         return {0u, nullptr};
3265       return {0u, &Hexagon::ModRegsRegClass};
3266     case 'q': // q0-q3
3267       switch (VT.getSizeInBits()) {
3268       default:
3269         return {0u, nullptr};
3270       case 64:
3271       case 128:
3272         return {0u, &Hexagon::HvxQRRegClass};
3273       }
3274       break;
3275     case 'v': // V0-V31
3276       switch (VT.getSizeInBits()) {
3277       default:
3278         return {0u, nullptr};
3279       case 512:
3280         return {0u, &Hexagon::HvxVRRegClass};
3281       case 1024:
3282         if (Subtarget.hasV60Ops() && Subtarget.useHVX128BOps())
3283           return {0u, &Hexagon::HvxVRRegClass};
3284         return {0u, &Hexagon::HvxWRRegClass};
3285       case 2048:
3286         return {0u, &Hexagon::HvxWRRegClass};
3287       }
3288       break;
3289     default:
3290       return {0u, nullptr};
3291     }
3292   }
3293 
3294   return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
3295 }
3296 
3297 /// isFPImmLegal - Returns true if the target can instruction select the
3298 /// specified FP immediate natively. If false, the legalizer will
3299 /// materialize the FP immediate as a load from a constant pool.
3300 bool HexagonTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
3301                                          bool ForCodeSize) const {
3302   return true;
3303 }
3304 
3305 /// isLegalAddressingMode - Return true if the addressing mode represented by
3306 /// AM is legal for this target, for a load/store of the specified type.
3307 bool HexagonTargetLowering::isLegalAddressingMode(const DataLayout &DL,
3308                                                   const AddrMode &AM, Type *Ty,
3309                                                   unsigned AS, Instruction *I) const {
3310   if (Ty->isSized()) {
3311     // When LSR detects uses of the same base address to access different
3312     // types (e.g. unions), it will assume a conservative type for these
3313     // uses:
3314     //   LSR Use: Kind=Address of void in addrspace(4294967295), ...
3315     // The type Ty passed here would then be "void". Skip the alignment
3316     // checks, but do not return false right away, since that confuses
3317     // LSR into crashing.
3318     Align A = DL.getABITypeAlign(Ty);
3319     // The base offset must be a multiple of the alignment.
3320     if (!isAligned(A, AM.BaseOffs))
3321       return false;
3322     // The shifted offset must fit in 11 bits.
3323     if (!isInt<11>(AM.BaseOffs >> Log2(A)))
3324       return false;
3325   }
3326 
3327   // No global is ever allowed as a base.
3328   if (AM.BaseGV)
3329     return false;
3330 
3331   int Scale = AM.Scale;
3332   if (Scale < 0)
3333     Scale = -Scale;
3334   switch (Scale) {
3335   case 0:  // No scale reg, "r+i", "r", or just "i".
3336     break;
3337   default: // No scaled addressing mode.
3338     return false;
3339   }
3340   return true;
3341 }
3342 
3343 /// Return true if folding a constant offset with the given GlobalAddress is
3344 /// legal.  It is frequently not legal in PIC relocation models.
3345 bool HexagonTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA)
3346       const {
3347   return HTM.getRelocationModel() == Reloc::Static;
3348 }
3349 
3350 /// isLegalICmpImmediate - Return true if the specified immediate is legal
3351 /// icmp immediate, that is the target has icmp instructions which can compare
3352 /// a register against the immediate without having to materialize the
3353 /// immediate into a register.
3354 bool HexagonTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
3355   return Imm >= -512 && Imm <= 511;
3356 }
3357 
3358 /// IsEligibleForTailCallOptimization - Check whether the call is eligible
3359 /// for tail call optimization. Targets which want to do tail call
3360 /// optimization should implement this function.
3361 bool HexagonTargetLowering::IsEligibleForTailCallOptimization(
3362                                  SDValue Callee,
3363                                  CallingConv::ID CalleeCC,
3364                                  bool IsVarArg,
3365                                  bool IsCalleeStructRet,
3366                                  bool IsCallerStructRet,
3367                                  const SmallVectorImpl<ISD::OutputArg> &Outs,
3368                                  const SmallVectorImpl<SDValue> &OutVals,
3369                                  const SmallVectorImpl<ISD::InputArg> &Ins,
3370                                  SelectionDAG& DAG) const {
3371   const Function &CallerF = DAG.getMachineFunction().getFunction();
3372   CallingConv::ID CallerCC = CallerF.getCallingConv();
3373   bool CCMatch = CallerCC == CalleeCC;
3374 
3375   // ***************************************************************************
3376   //  Look for obvious safe cases to perform tail call optimization that do not
3377   //  require ABI changes.
3378   // ***************************************************************************
3379 
3380   // If this is a tail call via a function pointer, then don't do it!
3381   if (!isa<GlobalAddressSDNode>(Callee) &&
3382       !isa<ExternalSymbolSDNode>(Callee)) {
3383     return false;
3384   }
3385 
3386   // Do not optimize if the calling conventions do not match and the conventions
3387   // used are not C or Fast.
3388   if (!CCMatch) {
3389     bool R = (CallerCC == CallingConv::C || CallerCC == CallingConv::Fast);
3390     bool E = (CalleeCC == CallingConv::C || CalleeCC == CallingConv::Fast);
3391     // If R & E, then ok.
3392     if (!R || !E)
3393       return false;
3394   }
3395 
3396   // Do not tail call optimize vararg calls.
3397   if (IsVarArg)
3398     return false;
3399 
3400   // Also avoid tail call optimization if either caller or callee uses struct
3401   // return semantics.
3402   if (IsCalleeStructRet || IsCallerStructRet)
3403     return false;
3404 
3405   // In addition to the cases above, we also disable Tail Call Optimization if
3406   // the calling convention code that at least one outgoing argument needs to
3407   // go on the stack. We cannot check that here because at this point that
3408   // information is not available.
3409   return true;
3410 }
3411 
3412 /// Returns the target specific optimal type for load and store operations as
3413 /// a result of memset, memcpy, and memmove lowering.
3414 ///
3415 /// If DstAlign is zero that means it's safe to destination alignment can
3416 /// satisfy any constraint. Similarly if SrcAlign is zero it means there isn't
3417 /// a need to check it against alignment requirement, probably because the
3418 /// source does not need to be loaded. If 'IsMemset' is true, that means it's
3419 /// expanding a memset. If 'ZeroMemset' is true, that means it's a memset of
3420 /// zero. 'MemcpyStrSrc' indicates whether the memcpy source is constant so it
3421 /// does not need to be loaded.  It returns EVT::Other if the type should be
3422 /// determined using generic target-independent logic.
3423 EVT HexagonTargetLowering::getOptimalMemOpType(
3424     const MemOp &Op, const AttributeList &FuncAttributes) const {
3425   if (Op.size() >= 8 && Op.isAligned(Align(8)))
3426     return MVT::i64;
3427   if (Op.size() >= 4 && Op.isAligned(Align(4)))
3428     return MVT::i32;
3429   if (Op.size() >= 2 && Op.isAligned(Align(2)))
3430     return MVT::i16;
3431   return MVT::Other;
3432 }
3433 
3434 bool HexagonTargetLowering::allowsMemoryAccess(
3435     LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
3436     Align Alignment, MachineMemOperand::Flags Flags, bool *Fast) const {
3437   MVT SVT = VT.getSimpleVT();
3438   if (Subtarget.isHVXVectorType(SVT, true))
3439     return allowsHvxMemoryAccess(SVT, Flags, Fast);
3440   return TargetLoweringBase::allowsMemoryAccess(
3441               Context, DL, VT, AddrSpace, Alignment, Flags, Fast);
3442 }
3443 
3444 bool HexagonTargetLowering::allowsMisalignedMemoryAccesses(
3445       EVT VT, unsigned AddrSpace, unsigned Alignment,
3446       MachineMemOperand::Flags Flags, bool *Fast) const {
3447   MVT SVT = VT.getSimpleVT();
3448   if (Subtarget.isHVXVectorType(SVT, true))
3449     return allowsHvxMisalignedMemoryAccesses(SVT, Flags, Fast);
3450   if (Fast)
3451     *Fast = false;
3452   return false;
3453 }
3454 
3455 std::pair<const TargetRegisterClass*, uint8_t>
3456 HexagonTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
3457       MVT VT) const {
3458   if (Subtarget.isHVXVectorType(VT, true)) {
3459     unsigned BitWidth = VT.getSizeInBits();
3460     unsigned VecWidth = Subtarget.getVectorLength() * 8;
3461 
3462     if (VT.getVectorElementType() == MVT::i1)
3463       return std::make_pair(&Hexagon::HvxQRRegClass, 1);
3464     if (BitWidth == VecWidth)
3465       return std::make_pair(&Hexagon::HvxVRRegClass, 1);
3466     assert(BitWidth == 2 * VecWidth);
3467     return std::make_pair(&Hexagon::HvxWRRegClass, 1);
3468   }
3469 
3470   return TargetLowering::findRepresentativeClass(TRI, VT);
3471 }
3472 
3473 bool HexagonTargetLowering::shouldReduceLoadWidth(SDNode *Load,
3474       ISD::LoadExtType ExtTy, EVT NewVT) const {
3475   // TODO: This may be worth removing. Check regression tests for diffs.
3476   if (!TargetLoweringBase::shouldReduceLoadWidth(Load, ExtTy, NewVT))
3477     return false;
3478 
3479   auto *L = cast<LoadSDNode>(Load);
3480   std::pair<SDValue,int> BO = getBaseAndOffset(L->getBasePtr());
3481   // Small-data object, do not shrink.
3482   if (BO.first.getOpcode() == HexagonISD::CONST32_GP)
3483     return false;
3484   if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(BO.first)) {
3485     auto &HTM = static_cast<const HexagonTargetMachine&>(getTargetMachine());
3486     const auto *GO = dyn_cast_or_null<const GlobalObject>(GA->getGlobal());
3487     return !GO || !HTM.getObjFileLowering()->isGlobalInSmallSection(GO, HTM);
3488   }
3489   return true;
3490 }
3491 
3492 Value *HexagonTargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr,
3493       AtomicOrdering Ord) const {
3494   BasicBlock *BB = Builder.GetInsertBlock();
3495   Module *M = BB->getParent()->getParent();
3496   auto PT = cast<PointerType>(Addr->getType());
3497   Type *Ty = PT->getElementType();
3498   unsigned SZ = Ty->getPrimitiveSizeInBits();
3499   assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported");
3500   Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_L2_loadw_locked
3501                                    : Intrinsic::hexagon_L4_loadd_locked;
3502   Function *Fn = Intrinsic::getDeclaration(M, IntID);
3503 
3504   PointerType *NewPtrTy
3505     = Builder.getIntNTy(SZ)->getPointerTo(PT->getAddressSpace());
3506   Addr = Builder.CreateBitCast(Addr, NewPtrTy);
3507 
3508   Value *Call = Builder.CreateCall(Fn, Addr, "larx");
3509 
3510   return Builder.CreateBitCast(Call, Ty);
3511 }
3512 
3513 /// Perform a store-conditional operation to Addr. Return the status of the
3514 /// store. This should be 0 if the store succeeded, non-zero otherwise.
3515 Value *HexagonTargetLowering::emitStoreConditional(IRBuilder<> &Builder,
3516       Value *Val, Value *Addr, AtomicOrdering Ord) const {
3517   BasicBlock *BB = Builder.GetInsertBlock();
3518   Module *M = BB->getParent()->getParent();
3519   Type *Ty = Val->getType();
3520   unsigned SZ = Ty->getPrimitiveSizeInBits();
3521 
3522   Type *CastTy = Builder.getIntNTy(SZ);
3523   assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported");
3524   Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_S2_storew_locked
3525                                    : Intrinsic::hexagon_S4_stored_locked;
3526   Function *Fn = Intrinsic::getDeclaration(M, IntID);
3527 
3528   unsigned AS = Addr->getType()->getPointerAddressSpace();
3529   Addr = Builder.CreateBitCast(Addr, CastTy->getPointerTo(AS));
3530   Val = Builder.CreateBitCast(Val, CastTy);
3531 
3532   Value *Call = Builder.CreateCall(Fn, {Addr, Val}, "stcx");
3533   Value *Cmp = Builder.CreateICmpEQ(Call, Builder.getInt32(0), "");
3534   Value *Ext = Builder.CreateZExt(Cmp, Type::getInt32Ty(M->getContext()));
3535   return Ext;
3536 }
3537 
3538 TargetLowering::AtomicExpansionKind
3539 HexagonTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
3540   // Do not expand loads and stores that don't exceed 64 bits.
3541   return LI->getType()->getPrimitiveSizeInBits() > 64
3542              ? AtomicExpansionKind::LLOnly
3543              : AtomicExpansionKind::None;
3544 }
3545 
3546 bool HexagonTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
3547   // Do not expand loads and stores that don't exceed 64 bits.
3548   return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() > 64;
3549 }
3550 
3551 TargetLowering::AtomicExpansionKind
3552 HexagonTargetLowering::shouldExpandAtomicCmpXchgInIR(
3553     AtomicCmpXchgInst *AI) const {
3554   return AtomicExpansionKind::LLSC;
3555 }
3556