xref: /freebsd/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp (revision 19261079b74319502c6ffa1249920079f0f69a72)
1 //===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the AArch64SelectionDAGInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "AArch64TargetMachine.h"
14 using namespace llvm;
15 
16 #define DEBUG_TYPE "aarch64-selectiondag-info"
17 
18 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset(
19     SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
20     SDValue Size, Align Alignment, bool isVolatile,
21     MachinePointerInfo DstPtrInfo) const {
22   // Check to see if there is a specialized entry-point for memory zeroing.
23   ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src);
24   ConstantSDNode *SizeValue = dyn_cast<ConstantSDNode>(Size);
25   const AArch64Subtarget &STI =
26       DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
27   const char *bzeroName = (V && V->isNullValue())
28       ? DAG.getTargetLoweringInfo().getLibcallName(RTLIB::BZERO) : nullptr;
29   // For small size (< 256), it is not beneficial to use bzero
30   // instead of memset.
31   if (bzeroName && (!SizeValue || SizeValue->getZExtValue() > 256)) {
32     const AArch64TargetLowering &TLI = *STI.getTargetLowering();
33 
34     EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout());
35     Type *IntPtrTy = Type::getInt8PtrTy(*DAG.getContext());
36     TargetLowering::ArgListTy Args;
37     TargetLowering::ArgListEntry Entry;
38     Entry.Node = Dst;
39     Entry.Ty = IntPtrTy;
40     Args.push_back(Entry);
41     Entry.Node = Size;
42     Args.push_back(Entry);
43     TargetLowering::CallLoweringInfo CLI(DAG);
44     CLI.setDebugLoc(dl)
45         .setChain(Chain)
46         .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
47                       DAG.getExternalSymbol(bzeroName, IntPtr),
48                       std::move(Args))
49         .setDiscardResult();
50     std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
51     return CallResult.second;
52   }
53   return SDValue();
54 }
55 bool AArch64SelectionDAGInfo::generateFMAsInMachineCombiner(
56     CodeGenOpt::Level OptLevel) const {
57   return OptLevel >= CodeGenOpt::Aggressive;
58 }
59 
60 static const int kSetTagLoopThreshold = 176;
61 
62 static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl,
63                                   SDValue Chain, SDValue Ptr, uint64_t ObjSize,
64                                   const MachineMemOperand *BaseMemOperand,
65                                   bool ZeroData) {
66   MachineFunction &MF = DAG.getMachineFunction();
67   unsigned ObjSizeScaled = ObjSize / 16;
68 
69   SDValue TagSrc = Ptr;
70   if (Ptr.getOpcode() == ISD::FrameIndex) {
71     int FI = cast<FrameIndexSDNode>(Ptr)->getIndex();
72     Ptr = DAG.getTargetFrameIndex(FI, MVT::i64);
73     // A frame index operand may end up as [SP + offset] => it is fine to use SP
74     // register as the tag source.
75     TagSrc = DAG.getRegister(AArch64::SP, MVT::i64);
76   }
77 
78   const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG;
79   const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G;
80 
81   SmallVector<SDValue, 8> OutChains;
82   unsigned OffsetScaled = 0;
83   while (OffsetScaled < ObjSizeScaled) {
84     if (ObjSizeScaled - OffsetScaled >= 2) {
85       SDValue AddrNode =
86           DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(OffsetScaled * 16), dl);
87       SDValue St = DAG.getMemIntrinsicNode(
88           OpCode2, dl, DAG.getVTList(MVT::Other),
89           {Chain, TagSrc, AddrNode},
90           MVT::v4i64,
91           MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16 * 2));
92       OffsetScaled += 2;
93       OutChains.push_back(St);
94       continue;
95     }
96 
97     if (ObjSizeScaled - OffsetScaled > 0) {
98       SDValue AddrNode =
99           DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(OffsetScaled * 16), dl);
100       SDValue St = DAG.getMemIntrinsicNode(
101           OpCode1, dl, DAG.getVTList(MVT::Other),
102           {Chain, TagSrc, AddrNode},
103           MVT::v2i64,
104           MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16));
105       OffsetScaled += 1;
106       OutChains.push_back(St);
107     }
108   }
109 
110   SDValue Res = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
111   return Res;
112 }
113 
114 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForSetTag(
115     SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr,
116     SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const {
117   uint64_t ObjSize = cast<ConstantSDNode>(Size)->getZExtValue();
118   assert(ObjSize % 16 == 0);
119 
120   MachineFunction &MF = DAG.getMachineFunction();
121   MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand(
122       DstPtrInfo, MachineMemOperand::MOStore, ObjSize, Align(16));
123 
124   bool UseSetTagRangeLoop =
125       kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold;
126   if (!UseSetTagRangeLoop)
127     return EmitUnrolledSetTag(DAG, dl, Chain, Addr, ObjSize, BaseMemOperand,
128                               ZeroData);
129 
130   const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other};
131 
132   unsigned Opcode;
133   if (Addr.getOpcode() == ISD::FrameIndex) {
134     int FI = cast<FrameIndexSDNode>(Addr)->getIndex();
135     Addr = DAG.getTargetFrameIndex(FI, MVT::i64);
136     Opcode = ZeroData ? AArch64::STZGloop : AArch64::STGloop;
137   } else {
138     Opcode = ZeroData ? AArch64::STZGloop_wback : AArch64::STGloop_wback;
139   }
140   SDValue Ops[] = {DAG.getTargetConstant(ObjSize, dl, MVT::i64), Addr, Chain};
141   SDNode *St = DAG.getMachineNode(Opcode, dl, ResTys, Ops);
142 
143   DAG.setNodeMemRefs(cast<MachineSDNode>(St), {BaseMemOperand});
144   return SDValue(St, 2);
145 }
146