1 //===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the AArch64SelectionDAGInfo class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "AArch64TargetMachine.h" 14 using namespace llvm; 15 16 #define DEBUG_TYPE "aarch64-selectiondag-info" 17 18 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset( 19 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, 20 SDValue Size, unsigned Align, bool isVolatile, 21 MachinePointerInfo DstPtrInfo) const { 22 // Check to see if there is a specialized entry-point for memory zeroing. 23 ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src); 24 ConstantSDNode *SizeValue = dyn_cast<ConstantSDNode>(Size); 25 const AArch64Subtarget &STI = 26 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>(); 27 const char *bzeroName = (V && V->isNullValue()) 28 ? DAG.getTargetLoweringInfo().getLibcallName(RTLIB::BZERO) : nullptr; 29 // For small size (< 256), it is not beneficial to use bzero 30 // instead of memset. 31 if (bzeroName && (!SizeValue || SizeValue->getZExtValue() > 256)) { 32 const AArch64TargetLowering &TLI = *STI.getTargetLowering(); 33 34 EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout()); 35 Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext()); 36 TargetLowering::ArgListTy Args; 37 TargetLowering::ArgListEntry Entry; 38 Entry.Node = Dst; 39 Entry.Ty = IntPtrTy; 40 Args.push_back(Entry); 41 Entry.Node = Size; 42 Args.push_back(Entry); 43 TargetLowering::CallLoweringInfo CLI(DAG); 44 CLI.setDebugLoc(dl) 45 .setChain(Chain) 46 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()), 47 DAG.getExternalSymbol(bzeroName, IntPtr), 48 std::move(Args)) 49 .setDiscardResult(); 50 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); 51 return CallResult.second; 52 } 53 return SDValue(); 54 } 55 bool AArch64SelectionDAGInfo::generateFMAsInMachineCombiner( 56 CodeGenOpt::Level OptLevel) const { 57 return OptLevel >= CodeGenOpt::Aggressive; 58 } 59 60 static const int kSetTagLoopThreshold = 176; 61 62 static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl, 63 SDValue Chain, SDValue Ptr, uint64_t ObjSize, 64 const MachineMemOperand *BaseMemOperand, 65 bool ZeroData) { 66 MachineFunction &MF = DAG.getMachineFunction(); 67 unsigned ObjSizeScaled = ObjSize / 16; 68 69 SDValue TagSrc = Ptr; 70 if (Ptr.getOpcode() == ISD::FrameIndex) { 71 int FI = cast<FrameIndexSDNode>(Ptr)->getIndex(); 72 Ptr = DAG.getTargetFrameIndex(FI, MVT::i64); 73 // A frame index operand may end up as [SP + offset] => it is fine to use SP 74 // register as the tag source. 75 TagSrc = DAG.getRegister(AArch64::SP, MVT::i64); 76 } 77 78 const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG; 79 const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G; 80 81 SmallVector<SDValue, 8> OutChains; 82 unsigned OffsetScaled = 0; 83 while (OffsetScaled < ObjSizeScaled) { 84 if (ObjSizeScaled - OffsetScaled >= 2) { 85 SDValue AddrNode = DAG.getMemBasePlusOffset(Ptr, OffsetScaled * 16, dl); 86 SDValue St = DAG.getMemIntrinsicNode( 87 OpCode2, dl, DAG.getVTList(MVT::Other), 88 {Chain, TagSrc, AddrNode}, 89 MVT::v4i64, 90 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16 * 2)); 91 OffsetScaled += 2; 92 OutChains.push_back(St); 93 continue; 94 } 95 96 if (ObjSizeScaled - OffsetScaled > 0) { 97 SDValue AddrNode = DAG.getMemBasePlusOffset(Ptr, OffsetScaled * 16, dl); 98 SDValue St = DAG.getMemIntrinsicNode( 99 OpCode1, dl, DAG.getVTList(MVT::Other), 100 {Chain, TagSrc, AddrNode}, 101 MVT::v2i64, 102 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16)); 103 OffsetScaled += 1; 104 OutChains.push_back(St); 105 } 106 } 107 108 SDValue Res = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); 109 return Res; 110 } 111 112 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForSetTag( 113 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr, 114 SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const { 115 uint64_t ObjSize = cast<ConstantSDNode>(Size)->getZExtValue(); 116 assert(ObjSize % 16 == 0); 117 118 MachineFunction &MF = DAG.getMachineFunction(); 119 MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand( 120 DstPtrInfo, MachineMemOperand::MOStore, ObjSize, 16); 121 122 bool UseSetTagRangeLoop = 123 kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold; 124 if (!UseSetTagRangeLoop) 125 return EmitUnrolledSetTag(DAG, dl, Chain, Addr, ObjSize, BaseMemOperand, 126 ZeroData); 127 128 if (ObjSize % 32 != 0) { 129 SDNode *St1 = DAG.getMachineNode( 130 ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex, dl, 131 {MVT::i64, MVT::Other}, 132 {Addr, Addr, DAG.getTargetConstant(1, dl, MVT::i64), Chain}); 133 DAG.setNodeMemRefs(cast<MachineSDNode>(St1), {BaseMemOperand}); 134 ObjSize -= 16; 135 Addr = SDValue(St1, 0); 136 Chain = SDValue(St1, 1); 137 } 138 139 const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other}; 140 SDValue Ops[] = {DAG.getConstant(ObjSize, dl, MVT::i64), Addr, Chain}; 141 SDNode *St = DAG.getMachineNode( 142 ZeroData ? AArch64::STZGloop : AArch64::STGloop, dl, ResTys, Ops); 143 144 DAG.setNodeMemRefs(cast<MachineSDNode>(St), {BaseMemOperand}); 145 return SDValue(St, 2); 146 } 147