1 //===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the AArch64SelectionDAGInfo class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "AArch64TargetMachine.h" 14 using namespace llvm; 15 16 #define DEBUG_TYPE "aarch64-selectiondag-info" 17 18 SDValue AArch64SelectionDAGInfo::EmitMOPS(AArch64ISD::NodeType SDOpcode, 19 SelectionDAG &DAG, const SDLoc &DL, 20 SDValue Chain, SDValue Dst, 21 SDValue SrcOrValue, SDValue Size, 22 Align Alignment, bool isVolatile, 23 MachinePointerInfo DstPtrInfo, 24 MachinePointerInfo SrcPtrInfo) const { 25 26 // Get the constant size of the copy/set. 27 uint64_t ConstSize = 0; 28 if (auto *C = dyn_cast<ConstantSDNode>(Size)) 29 ConstSize = C->getZExtValue(); 30 31 const bool IsSet = SDOpcode == AArch64ISD::MOPS_MEMSET || 32 SDOpcode == AArch64ISD::MOPS_MEMSET_TAGGING; 33 34 const auto MachineOpcode = [&]() { 35 switch (SDOpcode) { 36 case AArch64ISD::MOPS_MEMSET: 37 return AArch64::MOPSMemorySetPseudo; 38 case AArch64ISD::MOPS_MEMSET_TAGGING: 39 return AArch64::MOPSMemorySetTaggingPseudo; 40 case AArch64ISD::MOPS_MEMCOPY: 41 return AArch64::MOPSMemoryCopyPseudo; 42 case AArch64ISD::MOPS_MEMMOVE: 43 return AArch64::MOPSMemoryMovePseudo; 44 default: 45 llvm_unreachable("Unhandled MOPS ISD Opcode"); 46 } 47 }(); 48 49 MachineMemOperand::Flags Flags = MachineMemOperand::MOStore; 50 if (isVolatile) 51 Flags |= MachineMemOperand::MOVolatile; 52 if (!IsSet) 53 Flags |= MachineMemOperand::MOLoad; 54 55 MachineFunction &MF = DAG.getMachineFunction(); 56 57 auto *DstOp = 58 MF.getMachineMemOperand(DstPtrInfo, Flags, ConstSize, Alignment); 59 auto *SrcOp = 60 MF.getMachineMemOperand(SrcPtrInfo, Flags, ConstSize, Alignment); 61 62 if (IsSet) { 63 // Extend value to i64 if required 64 if (SrcOrValue.getValueType() != MVT::i64) 65 SrcOrValue = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, SrcOrValue); 66 SDValue Ops[] = {Dst, Size, SrcOrValue, Chain}; 67 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::Other}; 68 MachineSDNode *Node = DAG.getMachineNode(MachineOpcode, DL, ResultTys, Ops); 69 DAG.setNodeMemRefs(Node, {DstOp}); 70 return SDValue(Node, 2); 71 } else { 72 SDValue Ops[] = {Dst, SrcOrValue, Size, Chain}; 73 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::i64, MVT::Other}; 74 MachineSDNode *Node = DAG.getMachineNode(MachineOpcode, DL, ResultTys, Ops); 75 DAG.setNodeMemRefs(Node, {DstOp, SrcOp}); 76 return SDValue(Node, 3); 77 } 78 } 79 80 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemcpy( 81 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src, 82 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline, 83 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const { 84 const AArch64Subtarget &STI = 85 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>(); 86 if (STI.hasMOPS()) 87 return EmitMOPS(AArch64ISD::MOPS_MEMCOPY, DAG, DL, Chain, Dst, Src, Size, 88 Alignment, isVolatile, DstPtrInfo, SrcPtrInfo); 89 return SDValue(); 90 } 91 92 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset( 93 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, 94 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline, 95 MachinePointerInfo DstPtrInfo) const { 96 const AArch64Subtarget &STI = 97 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>(); 98 99 if (STI.hasMOPS()) { 100 return EmitMOPS(AArch64ISD::MOPS_MEMSET, DAG, dl, Chain, Dst, Src, Size, 101 Alignment, isVolatile, DstPtrInfo, MachinePointerInfo{}); 102 } 103 return SDValue(); 104 } 105 106 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemmove( 107 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, 108 SDValue Size, Align Alignment, bool isVolatile, 109 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const { 110 const AArch64Subtarget &STI = 111 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>(); 112 if (STI.hasMOPS()) { 113 return EmitMOPS(AArch64ISD::MOPS_MEMMOVE, DAG, dl, Chain, Dst, Src, Size, 114 Alignment, isVolatile, DstPtrInfo, SrcPtrInfo); 115 } 116 return SDValue(); 117 } 118 119 static const int kSetTagLoopThreshold = 176; 120 121 static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl, 122 SDValue Chain, SDValue Ptr, uint64_t ObjSize, 123 const MachineMemOperand *BaseMemOperand, 124 bool ZeroData) { 125 MachineFunction &MF = DAG.getMachineFunction(); 126 unsigned ObjSizeScaled = ObjSize / 16; 127 128 SDValue TagSrc = Ptr; 129 if (Ptr.getOpcode() == ISD::FrameIndex) { 130 int FI = cast<FrameIndexSDNode>(Ptr)->getIndex(); 131 Ptr = DAG.getTargetFrameIndex(FI, MVT::i64); 132 // A frame index operand may end up as [SP + offset] => it is fine to use SP 133 // register as the tag source. 134 TagSrc = DAG.getRegister(AArch64::SP, MVT::i64); 135 } 136 137 const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG; 138 const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G; 139 140 SmallVector<SDValue, 8> OutChains; 141 unsigned OffsetScaled = 0; 142 while (OffsetScaled < ObjSizeScaled) { 143 if (ObjSizeScaled - OffsetScaled >= 2) { 144 SDValue AddrNode = 145 DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(OffsetScaled * 16), dl); 146 SDValue St = DAG.getMemIntrinsicNode( 147 OpCode2, dl, DAG.getVTList(MVT::Other), 148 {Chain, TagSrc, AddrNode}, 149 MVT::v4i64, 150 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16 * 2)); 151 OffsetScaled += 2; 152 OutChains.push_back(St); 153 continue; 154 } 155 156 if (ObjSizeScaled - OffsetScaled > 0) { 157 SDValue AddrNode = 158 DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(OffsetScaled * 16), dl); 159 SDValue St = DAG.getMemIntrinsicNode( 160 OpCode1, dl, DAG.getVTList(MVT::Other), 161 {Chain, TagSrc, AddrNode}, 162 MVT::v2i64, 163 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16)); 164 OffsetScaled += 1; 165 OutChains.push_back(St); 166 } 167 } 168 169 SDValue Res = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); 170 return Res; 171 } 172 173 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForSetTag( 174 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr, 175 SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const { 176 uint64_t ObjSize = cast<ConstantSDNode>(Size)->getZExtValue(); 177 assert(ObjSize % 16 == 0); 178 179 MachineFunction &MF = DAG.getMachineFunction(); 180 MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand( 181 DstPtrInfo, MachineMemOperand::MOStore, ObjSize, Align(16)); 182 183 bool UseSetTagRangeLoop = 184 kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold; 185 if (!UseSetTagRangeLoop) 186 return EmitUnrolledSetTag(DAG, dl, Chain, Addr, ObjSize, BaseMemOperand, 187 ZeroData); 188 189 const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other}; 190 191 unsigned Opcode; 192 if (Addr.getOpcode() == ISD::FrameIndex) { 193 int FI = cast<FrameIndexSDNode>(Addr)->getIndex(); 194 Addr = DAG.getTargetFrameIndex(FI, MVT::i64); 195 Opcode = ZeroData ? AArch64::STZGloop : AArch64::STGloop; 196 } else { 197 Opcode = ZeroData ? AArch64::STZGloop_wback : AArch64::STGloop_wback; 198 } 199 SDValue Ops[] = {DAG.getTargetConstant(ObjSize, dl, MVT::i64), Addr, Chain}; 200 SDNode *St = DAG.getMachineNode(Opcode, dl, ResTys, Ops); 201 202 DAG.setNodeMemRefs(cast<MachineSDNode>(St), {BaseMemOperand}); 203 return SDValue(St, 2); 204 } 205