1 //===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the AArch64SelectionDAGInfo class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "AArch64TargetMachine.h" 14 using namespace llvm; 15 16 #define DEBUG_TYPE "aarch64-selectiondag-info" 17 18 SDValue AArch64SelectionDAGInfo::EmitMOPS(AArch64ISD::NodeType SDOpcode, 19 SelectionDAG &DAG, const SDLoc &DL, 20 SDValue Chain, SDValue Dst, 21 SDValue SrcOrValue, SDValue Size, 22 Align Alignment, bool isVolatile, 23 MachinePointerInfo DstPtrInfo, 24 MachinePointerInfo SrcPtrInfo) const { 25 26 // Get the constant size of the copy/set. 27 uint64_t ConstSize = 0; 28 if (auto *C = dyn_cast<ConstantSDNode>(Size)) 29 ConstSize = C->getZExtValue(); 30 31 const bool IsSet = SDOpcode == AArch64ISD::MOPS_MEMSET || 32 SDOpcode == AArch64ISD::MOPS_MEMSET_TAGGING; 33 34 const auto MachineOpcode = [&]() { 35 switch (SDOpcode) { 36 case AArch64ISD::MOPS_MEMSET: 37 return AArch64::MOPSMemorySetPseudo; 38 case AArch64ISD::MOPS_MEMSET_TAGGING: 39 return AArch64::MOPSMemorySetTaggingPseudo; 40 case AArch64ISD::MOPS_MEMCOPY: 41 return AArch64::MOPSMemoryCopyPseudo; 42 case AArch64ISD::MOPS_MEMMOVE: 43 return AArch64::MOPSMemoryMovePseudo; 44 default: 45 llvm_unreachable("Unhandled MOPS ISD Opcode"); 46 } 47 }(); 48 49 MachineFunction &MF = DAG.getMachineFunction(); 50 51 auto Vol = 52 isVolatile ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone; 53 auto DstFlags = MachineMemOperand::MOStore | Vol; 54 auto *DstOp = 55 MF.getMachineMemOperand(DstPtrInfo, DstFlags, ConstSize, Alignment); 56 57 if (IsSet) { 58 // Extend value to i64, if required. 59 if (SrcOrValue.getValueType() != MVT::i64) 60 SrcOrValue = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, SrcOrValue); 61 SDValue Ops[] = {Dst, Size, SrcOrValue, Chain}; 62 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::Other}; 63 MachineSDNode *Node = DAG.getMachineNode(MachineOpcode, DL, ResultTys, Ops); 64 DAG.setNodeMemRefs(Node, {DstOp}); 65 return SDValue(Node, 2); 66 } else { 67 SDValue Ops[] = {Dst, SrcOrValue, Size, Chain}; 68 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::i64, MVT::Other}; 69 MachineSDNode *Node = DAG.getMachineNode(MachineOpcode, DL, ResultTys, Ops); 70 71 auto SrcFlags = MachineMemOperand::MOLoad | Vol; 72 auto *SrcOp = 73 MF.getMachineMemOperand(SrcPtrInfo, SrcFlags, ConstSize, Alignment); 74 DAG.setNodeMemRefs(Node, {DstOp, SrcOp}); 75 return SDValue(Node, 3); 76 } 77 } 78 79 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemcpy( 80 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src, 81 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline, 82 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const { 83 const AArch64Subtarget &STI = 84 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>(); 85 if (STI.hasMOPS()) 86 return EmitMOPS(AArch64ISD::MOPS_MEMCOPY, DAG, DL, Chain, Dst, Src, Size, 87 Alignment, isVolatile, DstPtrInfo, SrcPtrInfo); 88 return SDValue(); 89 } 90 91 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset( 92 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, 93 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline, 94 MachinePointerInfo DstPtrInfo) const { 95 const AArch64Subtarget &STI = 96 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>(); 97 98 if (STI.hasMOPS()) { 99 return EmitMOPS(AArch64ISD::MOPS_MEMSET, DAG, dl, Chain, Dst, Src, Size, 100 Alignment, isVolatile, DstPtrInfo, MachinePointerInfo{}); 101 } 102 return SDValue(); 103 } 104 105 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemmove( 106 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, 107 SDValue Size, Align Alignment, bool isVolatile, 108 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const { 109 const AArch64Subtarget &STI = 110 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>(); 111 if (STI.hasMOPS()) { 112 return EmitMOPS(AArch64ISD::MOPS_MEMMOVE, DAG, dl, Chain, Dst, Src, Size, 113 Alignment, isVolatile, DstPtrInfo, SrcPtrInfo); 114 } 115 return SDValue(); 116 } 117 118 static const int kSetTagLoopThreshold = 176; 119 120 static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl, 121 SDValue Chain, SDValue Ptr, uint64_t ObjSize, 122 const MachineMemOperand *BaseMemOperand, 123 bool ZeroData) { 124 MachineFunction &MF = DAG.getMachineFunction(); 125 unsigned ObjSizeScaled = ObjSize / 16; 126 127 SDValue TagSrc = Ptr; 128 if (Ptr.getOpcode() == ISD::FrameIndex) { 129 int FI = cast<FrameIndexSDNode>(Ptr)->getIndex(); 130 Ptr = DAG.getTargetFrameIndex(FI, MVT::i64); 131 // A frame index operand may end up as [SP + offset] => it is fine to use SP 132 // register as the tag source. 133 TagSrc = DAG.getRegister(AArch64::SP, MVT::i64); 134 } 135 136 const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG; 137 const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G; 138 139 SmallVector<SDValue, 8> OutChains; 140 unsigned OffsetScaled = 0; 141 while (OffsetScaled < ObjSizeScaled) { 142 if (ObjSizeScaled - OffsetScaled >= 2) { 143 SDValue AddrNode = DAG.getMemBasePlusOffset( 144 Ptr, TypeSize::getFixed(OffsetScaled * 16), dl); 145 SDValue St = DAG.getMemIntrinsicNode( 146 OpCode2, dl, DAG.getVTList(MVT::Other), 147 {Chain, TagSrc, AddrNode}, 148 MVT::v4i64, 149 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16 * 2)); 150 OffsetScaled += 2; 151 OutChains.push_back(St); 152 continue; 153 } 154 155 if (ObjSizeScaled - OffsetScaled > 0) { 156 SDValue AddrNode = DAG.getMemBasePlusOffset( 157 Ptr, TypeSize::getFixed(OffsetScaled * 16), dl); 158 SDValue St = DAG.getMemIntrinsicNode( 159 OpCode1, dl, DAG.getVTList(MVT::Other), 160 {Chain, TagSrc, AddrNode}, 161 MVT::v2i64, 162 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16)); 163 OffsetScaled += 1; 164 OutChains.push_back(St); 165 } 166 } 167 168 SDValue Res = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); 169 return Res; 170 } 171 172 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForSetTag( 173 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr, 174 SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const { 175 uint64_t ObjSize = Size->getAsZExtVal(); 176 assert(ObjSize % 16 == 0); 177 178 MachineFunction &MF = DAG.getMachineFunction(); 179 MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand( 180 DstPtrInfo, MachineMemOperand::MOStore, ObjSize, Align(16)); 181 182 bool UseSetTagRangeLoop = 183 kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold; 184 if (!UseSetTagRangeLoop) 185 return EmitUnrolledSetTag(DAG, dl, Chain, Addr, ObjSize, BaseMemOperand, 186 ZeroData); 187 188 const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other}; 189 190 unsigned Opcode; 191 if (Addr.getOpcode() == ISD::FrameIndex) { 192 int FI = cast<FrameIndexSDNode>(Addr)->getIndex(); 193 Addr = DAG.getTargetFrameIndex(FI, MVT::i64); 194 Opcode = ZeroData ? AArch64::STZGloop : AArch64::STGloop; 195 } else { 196 Opcode = ZeroData ? AArch64::STZGloop_wback : AArch64::STGloop_wback; 197 } 198 SDValue Ops[] = {DAG.getTargetConstant(ObjSize, dl, MVT::i64), Addr, Chain}; 199 SDNode *St = DAG.getMachineNode(Opcode, dl, ResTys, Ops); 200 201 DAG.setNodeMemRefs(cast<MachineSDNode>(St), {BaseMemOperand}); 202 return SDValue(St, 2); 203 } 204