1 //===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the AArch64SelectionDAGInfo class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "AArch64TargetMachine.h" 14 using namespace llvm; 15 16 #define DEBUG_TYPE "aarch64-selectiondag-info" 17 18 SDValue AArch64SelectionDAGInfo::EmitMOPS(AArch64ISD::NodeType SDOpcode, 19 SelectionDAG &DAG, const SDLoc &DL, 20 SDValue Chain, SDValue Dst, 21 SDValue SrcOrValue, SDValue Size, 22 Align Alignment, bool isVolatile, 23 MachinePointerInfo DstPtrInfo, 24 MachinePointerInfo SrcPtrInfo) const { 25 26 // Get the constant size of the copy/set. 27 uint64_t ConstSize = 0; 28 if (auto *C = dyn_cast<ConstantSDNode>(Size)) 29 ConstSize = C->getZExtValue(); 30 31 const bool IsSet = SDOpcode == AArch64ISD::MOPS_MEMSET || 32 SDOpcode == AArch64ISD::MOPS_MEMSET_TAGGING; 33 34 const auto MachineOpcode = [&]() { 35 switch (SDOpcode) { 36 case AArch64ISD::MOPS_MEMSET: 37 return AArch64::MOPSMemorySetPseudo; 38 case AArch64ISD::MOPS_MEMSET_TAGGING: 39 return AArch64::MOPSMemorySetTaggingPseudo; 40 case AArch64ISD::MOPS_MEMCOPY: 41 return AArch64::MOPSMemoryCopyPseudo; 42 case AArch64ISD::MOPS_MEMMOVE: 43 return AArch64::MOPSMemoryMovePseudo; 44 default: 45 llvm_unreachable("Unhandled MOPS ISD Opcode"); 46 } 47 }(); 48 49 MachineMemOperand::Flags Flags = MachineMemOperand::MOStore; 50 if (isVolatile) 51 Flags |= MachineMemOperand::MOVolatile; 52 if (!IsSet) 53 Flags |= MachineMemOperand::MOLoad; 54 55 MachineFunction &MF = DAG.getMachineFunction(); 56 57 auto *DstOp = 58 MF.getMachineMemOperand(DstPtrInfo, Flags, ConstSize, Alignment); 59 auto *SrcOp = 60 MF.getMachineMemOperand(SrcPtrInfo, Flags, ConstSize, Alignment); 61 62 if (IsSet) { 63 // Extend value to i64 if required 64 if (SrcOrValue.getValueType() != MVT::i64) 65 SrcOrValue = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, SrcOrValue); 66 SDValue Ops[] = {Dst, Size, SrcOrValue, Chain}; 67 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::Other}; 68 MachineSDNode *Node = DAG.getMachineNode(MachineOpcode, DL, ResultTys, Ops); 69 DAG.setNodeMemRefs(Node, {DstOp}); 70 return SDValue(Node, 2); 71 } else { 72 SDValue Ops[] = {Dst, SrcOrValue, Size, Chain}; 73 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::i64, MVT::Other}; 74 MachineSDNode *Node = DAG.getMachineNode(MachineOpcode, DL, ResultTys, Ops); 75 DAG.setNodeMemRefs(Node, {DstOp, SrcOp}); 76 return SDValue(Node, 3); 77 } 78 } 79 80 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemcpy( 81 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src, 82 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline, 83 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const { 84 const AArch64Subtarget &STI = 85 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>(); 86 if (STI.hasMOPS()) 87 return EmitMOPS(AArch64ISD::MOPS_MEMCOPY, DAG, DL, Chain, Dst, Src, Size, 88 Alignment, isVolatile, DstPtrInfo, SrcPtrInfo); 89 return SDValue(); 90 } 91 92 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset( 93 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, 94 SDValue Size, Align Alignment, bool isVolatile, 95 MachinePointerInfo DstPtrInfo) const { 96 const AArch64Subtarget &STI = 97 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>(); 98 99 if (STI.hasMOPS()) { 100 return EmitMOPS(AArch64ISD::MOPS_MEMSET, DAG, dl, Chain, Dst, Src, Size, 101 Alignment, isVolatile, DstPtrInfo, MachinePointerInfo{}); 102 } 103 104 // Check to see if there is a specialized entry-point for memory zeroing. 105 ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src); 106 ConstantSDNode *SizeValue = dyn_cast<ConstantSDNode>(Size); 107 const char *bzeroName = 108 (V && V->isZero()) 109 ? DAG.getTargetLoweringInfo().getLibcallName(RTLIB::BZERO) 110 : nullptr; 111 // For small size (< 256), it is not beneficial to use bzero 112 // instead of memset. 113 if (bzeroName && (!SizeValue || SizeValue->getZExtValue() > 256)) { 114 const AArch64TargetLowering &TLI = *STI.getTargetLowering(); 115 116 EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout()); 117 Type *IntPtrTy = Type::getInt8PtrTy(*DAG.getContext()); 118 TargetLowering::ArgListTy Args; 119 TargetLowering::ArgListEntry Entry; 120 Entry.Node = Dst; 121 Entry.Ty = IntPtrTy; 122 Args.push_back(Entry); 123 Entry.Node = Size; 124 Args.push_back(Entry); 125 TargetLowering::CallLoweringInfo CLI(DAG); 126 CLI.setDebugLoc(dl) 127 .setChain(Chain) 128 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()), 129 DAG.getExternalSymbol(bzeroName, IntPtr), 130 std::move(Args)) 131 .setDiscardResult(); 132 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); 133 return CallResult.second; 134 } 135 return SDValue(); 136 } 137 138 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemmove( 139 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, 140 SDValue Size, Align Alignment, bool isVolatile, 141 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const { 142 const AArch64Subtarget &STI = 143 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>(); 144 if (STI.hasMOPS()) { 145 return EmitMOPS(AArch64ISD::MOPS_MEMMOVE, DAG, dl, Chain, Dst, Src, Size, 146 Alignment, isVolatile, DstPtrInfo, SrcPtrInfo); 147 } 148 return SDValue(); 149 } 150 151 static const int kSetTagLoopThreshold = 176; 152 153 static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl, 154 SDValue Chain, SDValue Ptr, uint64_t ObjSize, 155 const MachineMemOperand *BaseMemOperand, 156 bool ZeroData) { 157 MachineFunction &MF = DAG.getMachineFunction(); 158 unsigned ObjSizeScaled = ObjSize / 16; 159 160 SDValue TagSrc = Ptr; 161 if (Ptr.getOpcode() == ISD::FrameIndex) { 162 int FI = cast<FrameIndexSDNode>(Ptr)->getIndex(); 163 Ptr = DAG.getTargetFrameIndex(FI, MVT::i64); 164 // A frame index operand may end up as [SP + offset] => it is fine to use SP 165 // register as the tag source. 166 TagSrc = DAG.getRegister(AArch64::SP, MVT::i64); 167 } 168 169 const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG; 170 const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G; 171 172 SmallVector<SDValue, 8> OutChains; 173 unsigned OffsetScaled = 0; 174 while (OffsetScaled < ObjSizeScaled) { 175 if (ObjSizeScaled - OffsetScaled >= 2) { 176 SDValue AddrNode = 177 DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(OffsetScaled * 16), dl); 178 SDValue St = DAG.getMemIntrinsicNode( 179 OpCode2, dl, DAG.getVTList(MVT::Other), 180 {Chain, TagSrc, AddrNode}, 181 MVT::v4i64, 182 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16 * 2)); 183 OffsetScaled += 2; 184 OutChains.push_back(St); 185 continue; 186 } 187 188 if (ObjSizeScaled - OffsetScaled > 0) { 189 SDValue AddrNode = 190 DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(OffsetScaled * 16), dl); 191 SDValue St = DAG.getMemIntrinsicNode( 192 OpCode1, dl, DAG.getVTList(MVT::Other), 193 {Chain, TagSrc, AddrNode}, 194 MVT::v2i64, 195 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16)); 196 OffsetScaled += 1; 197 OutChains.push_back(St); 198 } 199 } 200 201 SDValue Res = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); 202 return Res; 203 } 204 205 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForSetTag( 206 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr, 207 SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const { 208 uint64_t ObjSize = cast<ConstantSDNode>(Size)->getZExtValue(); 209 assert(ObjSize % 16 == 0); 210 211 MachineFunction &MF = DAG.getMachineFunction(); 212 MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand( 213 DstPtrInfo, MachineMemOperand::MOStore, ObjSize, Align(16)); 214 215 bool UseSetTagRangeLoop = 216 kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold; 217 if (!UseSetTagRangeLoop) 218 return EmitUnrolledSetTag(DAG, dl, Chain, Addr, ObjSize, BaseMemOperand, 219 ZeroData); 220 221 const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other}; 222 223 unsigned Opcode; 224 if (Addr.getOpcode() == ISD::FrameIndex) { 225 int FI = cast<FrameIndexSDNode>(Addr)->getIndex(); 226 Addr = DAG.getTargetFrameIndex(FI, MVT::i64); 227 Opcode = ZeroData ? AArch64::STZGloop : AArch64::STGloop; 228 } else { 229 Opcode = ZeroData ? AArch64::STZGloop_wback : AArch64::STGloop_wback; 230 } 231 SDValue Ops[] = {DAG.getTargetConstant(ObjSize, dl, MVT::i64), Addr, Chain}; 232 SDNode *St = DAG.getMachineNode(Opcode, dl, ResTys, Ops); 233 234 DAG.setNodeMemRefs(cast<MachineSDNode>(St), {BaseMemOperand}); 235 return SDValue(St, 2); 236 } 237