// // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file contains a pass that performs optimization on SIMD instructions // with high latency by splitting them into more efficient series of // instructions. // // 1. Rewrite certain SIMD instructions with vector element due to their // inefficiency on some targets. // // For example: // fmla v0.4s, v1.4s, v2.s[1] // // Is rewritten into: // dup v3.4s, v2.s[1] // fmla v0.4s, v1.4s, v3.4s // // 2. Rewrite interleaved memory access instructions due to their // inefficiency on some targets. // // For example: // st2 {v0.4s, v1.4s}, addr // // Is rewritten into: // zip1 v2.4s, v0.4s, v1.4s // zip2 v3.4s, v0.4s, v1.4s // stp q2, q3, addr // //===----------------------------------------------------------------------===// #include "AArch64InstrInfo.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringRef.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetSchedule.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/MC/MCInstrDesc.h" #include "llvm/MC/MCSchedule.h" #include "llvm/Pass.h" #include #include using namespace llvm; #define DEBUG_TYPE "aarch64-simdinstr-opt" STATISTIC(NumModifiedInstr, "Number of SIMD instructions modified"); #define AARCH64_VECTOR_BY_ELEMENT_OPT_NAME \ "AArch64 SIMD instructions optimization pass" namespace { struct AArch64SIMDInstrOpt : public MachineFunctionPass { static char ID; const TargetInstrInfo *TII; MachineRegisterInfo *MRI; TargetSchedModel SchedModel; // The two maps below are used to cache decisions instead of recomputing: // This is used to cache instruction replacement decisions within function // units and across function units. std::map, bool> SIMDInstrTable; // This is used to cache the decision of whether to leave the interleaved // store instructions replacement pass early or not for a particular target. std::unordered_map InterlEarlyExit; typedef enum { VectorElem, Interleave } Subpass; // Instruction represented by OrigOpc is replaced by instructions in ReplOpc. struct InstReplInfo { unsigned OrigOpc; std::vector ReplOpc; const TargetRegisterClass RC; }; #define RuleST2(OpcOrg, OpcR0, OpcR1, OpcR2, RC) \ {OpcOrg, {OpcR0, OpcR1, OpcR2}, RC} #define RuleST4(OpcOrg, OpcR0, OpcR1, OpcR2, OpcR3, OpcR4, OpcR5, OpcR6, \ OpcR7, OpcR8, OpcR9, RC) \ {OpcOrg, \ {OpcR0, OpcR1, OpcR2, OpcR3, OpcR4, OpcR5, OpcR6, OpcR7, OpcR8, OpcR9}, RC} // The Instruction Replacement Table: std::vector IRT = { // ST2 instructions RuleST2(AArch64::ST2Twov2d, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, AArch64::STPQi, AArch64::FPR128RegClass), RuleST2(AArch64::ST2Twov4s, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, AArch64::STPQi, AArch64::FPR128RegClass), RuleST2(AArch64::ST2Twov2s, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, AArch64::STPDi, AArch64::FPR64RegClass), RuleST2(AArch64::ST2Twov8h, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, AArch64::STPQi, AArch64::FPR128RegClass), RuleST2(AArch64::ST2Twov4h, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, AArch64::STPDi, AArch64::FPR64RegClass), RuleST2(AArch64::ST2Twov16b, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, AArch64::STPQi, AArch64::FPR128RegClass), RuleST2(AArch64::ST2Twov8b, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, AArch64::STPDi, AArch64::FPR64RegClass), // ST4 instructions RuleST4(AArch64::ST4Fourv2d, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass), RuleST4(AArch64::ST4Fourv4s, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass), RuleST4(AArch64::ST4Fourv2s, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass), RuleST4(AArch64::ST4Fourv8h, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass), RuleST4(AArch64::ST4Fourv4h, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass), RuleST4(AArch64::ST4Fourv16b, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass), RuleST4(AArch64::ST4Fourv8b, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass) }; // A costly instruction is replaced in this work by N efficient instructions // The maximum of N is curently 10 and it is for ST4 case. static const unsigned MaxNumRepl = 10; AArch64SIMDInstrOpt() : MachineFunctionPass(ID) { initializeAArch64SIMDInstrOptPass(*PassRegistry::getPassRegistry()); } /// Based only on latency of instructions, determine if it is cost efficient /// to replace the instruction InstDesc by the instructions stored in the /// array InstDescRepl. /// Return true if replacement is expected to be faster. bool shouldReplaceInst(MachineFunction *MF, const MCInstrDesc *InstDesc, SmallVectorImpl &ReplInstrMCID); /// Determine if we need to exit the instruction replacement optimization /// passes early. This makes sure that no compile time is spent in this pass /// for targets with no need for any of these optimizations. /// Return true if early exit of the pass is recommended. bool shouldExitEarly(MachineFunction *MF, Subpass SP); /// Check whether an equivalent DUP instruction has already been /// created or not. /// Return true when the DUP instruction already exists. In this case, /// DestReg will point to the destination of the already created DUP. bool reuseDUP(MachineInstr &MI, unsigned DupOpcode, unsigned SrcReg, unsigned LaneNumber, unsigned *DestReg) const; /// Certain SIMD instructions with vector element operand are not efficient. /// Rewrite them into SIMD instructions with vector operands. This rewrite /// is driven by the latency of the instructions. /// Return true if the SIMD instruction is modified. bool optimizeVectElement(MachineInstr &MI); /// Process The REG_SEQUENCE instruction, and extract the source /// operands of the ST2/4 instruction from it. /// Example of such instructions. /// %dest = REG_SEQUENCE %st2_src1, dsub0, %st2_src2, dsub1; /// Return true when the instruction is processed successfully. bool processSeqRegInst(MachineInstr *DefiningMI, unsigned* StReg, unsigned* StRegKill, unsigned NumArg) const; /// Load/Store Interleaving instructions are not always beneficial. /// Replace them by ZIP instructionand classical load/store. /// Return true if the SIMD instruction is modified. bool optimizeLdStInterleave(MachineInstr &MI); /// Return the number of useful source registers for this /// instruction (2 for ST2 and 4 for ST4). unsigned determineSrcReg(MachineInstr &MI) const; bool runOnMachineFunction(MachineFunction &Fn) override; StringRef getPassName() const override { return AARCH64_VECTOR_BY_ELEMENT_OPT_NAME; } }; char AArch64SIMDInstrOpt::ID = 0; } // end anonymous namespace INITIALIZE_PASS(AArch64SIMDInstrOpt, "aarch64-simdinstr-opt", AARCH64_VECTOR_BY_ELEMENT_OPT_NAME, false, false) /// Based only on latency of instructions, determine if it is cost efficient /// to replace the instruction InstDesc by the instructions stored in the /// array InstDescRepl. /// Return true if replacement is expected to be faster. bool AArch64SIMDInstrOpt:: shouldReplaceInst(MachineFunction *MF, const MCInstrDesc *InstDesc, SmallVectorImpl &InstDescRepl) { // Check if replacement decision is already available in the cached table. // if so, return it. std::string Subtarget = std::string(SchedModel.getSubtargetInfo()->getCPU()); auto InstID = std::make_pair(InstDesc->getOpcode(), Subtarget); auto It = SIMDInstrTable.find(InstID); if (It != SIMDInstrTable.end()) return It->second; unsigned SCIdx = InstDesc->getSchedClass(); const MCSchedClassDesc *SCDesc = SchedModel.getMCSchedModel()->getSchedClassDesc(SCIdx); // If a target does not define resources for the instructions // of interest, then return false for no replacement. const MCSchedClassDesc *SCDescRepl; if (!SCDesc->isValid() || SCDesc->isVariant()) { SIMDInstrTable[InstID] = false; return false; } for (const auto *IDesc : InstDescRepl) { SCDescRepl = SchedModel.getMCSchedModel()->getSchedClassDesc( IDesc->getSchedClass()); if (!SCDescRepl->isValid() || SCDescRepl->isVariant()) { SIMDInstrTable[InstID] = false; return false; } } // Replacement cost. unsigned ReplCost = 0; for (const auto *IDesc :InstDescRepl) ReplCost += SchedModel.computeInstrLatency(IDesc->getOpcode()); if (SchedModel.computeInstrLatency(InstDesc->getOpcode()) > ReplCost) { SIMDInstrTable[InstID] = true; return true; } else { SIMDInstrTable[InstID] = false; return false; } } /// Determine if we need to exit this pass for a kind of instruction replacement /// early. This makes sure that no compile time is spent in this pass for /// targets with no need for any of these optimizations beyond performing this /// check. /// Return true if early exit of this pass for a kind of instruction /// replacement is recommended for a target. bool AArch64SIMDInstrOpt::shouldExitEarly(MachineFunction *MF, Subpass SP) { const MCInstrDesc* OriginalMCID; SmallVector ReplInstrMCID; switch (SP) { // For this optimization, check by comparing the latency of a representative // instruction to that of the replacement instructions. // TODO: check for all concerned instructions. case VectorElem: OriginalMCID = &TII->get(AArch64::FMLAv4i32_indexed); ReplInstrMCID.push_back(&TII->get(AArch64::DUPv4i32lane)); ReplInstrMCID.push_back(&TII->get(AArch64::FMLAv4f32)); if (shouldReplaceInst(MF, OriginalMCID, ReplInstrMCID)) return false; break; // For this optimization, check for all concerned instructions. case Interleave: std::string Subtarget = std::string(SchedModel.getSubtargetInfo()->getCPU()); auto It = InterlEarlyExit.find(Subtarget); if (It != InterlEarlyExit.end()) return It->second; for (auto &I : IRT) { OriginalMCID = &TII->get(I.OrigOpc); for (auto &Repl : I.ReplOpc) ReplInstrMCID.push_back(&TII->get(Repl)); if (shouldReplaceInst(MF, OriginalMCID, ReplInstrMCID)) { InterlEarlyExit[Subtarget] = false; return false; } ReplInstrMCID.clear(); } InterlEarlyExit[Subtarget] = true; break; } return true; } /// Check whether an equivalent DUP instruction has already been /// created or not. /// Return true when the DUP instruction already exists. In this case, /// DestReg will point to the destination of the already created DUP. bool AArch64SIMDInstrOpt::reuseDUP(MachineInstr &MI, unsigned DupOpcode, unsigned SrcReg, unsigned LaneNumber, unsigned *DestReg) const { for (MachineBasicBlock::iterator MII = MI, MIE = MI.getParent()->begin(); MII != MIE;) { MII--; MachineInstr *CurrentMI = &*MII; if (CurrentMI->getOpcode() == DupOpcode && CurrentMI->getNumOperands() == 3 && CurrentMI->getOperand(1).getReg() == SrcReg && CurrentMI->getOperand(2).getImm() == LaneNumber) { *DestReg = CurrentMI->getOperand(0).getReg(); return true; } } return false; } /// Certain SIMD instructions with vector element operand are not efficient. /// Rewrite them into SIMD instructions with vector operands. This rewrite /// is driven by the latency of the instructions. /// The instruction of concerns are for the time being FMLA, FMLS, FMUL, /// and FMULX and hence they are hardcoded. /// /// For example: /// fmla v0.4s, v1.4s, v2.s[1] /// /// Is rewritten into /// dup v3.4s, v2.s[1] // DUP not necessary if redundant /// fmla v0.4s, v1.4s, v3.4s /// /// Return true if the SIMD instruction is modified. bool AArch64SIMDInstrOpt::optimizeVectElement(MachineInstr &MI) { const MCInstrDesc *MulMCID, *DupMCID; const TargetRegisterClass *RC = &AArch64::FPR128RegClass; switch (MI.getOpcode()) { default: return false; // 4X32 instructions case AArch64::FMLAv4i32_indexed: DupMCID = &TII->get(AArch64::DUPv4i32lane); MulMCID = &TII->get(AArch64::FMLAv4f32); break; case AArch64::FMLSv4i32_indexed: DupMCID = &TII->get(AArch64::DUPv4i32lane); MulMCID = &TII->get(AArch64::FMLSv4f32); break; case AArch64::FMULXv4i32_indexed: DupMCID = &TII->get(AArch64::DUPv4i32lane); MulMCID = &TII->get(AArch64::FMULXv4f32); break; case AArch64::FMULv4i32_indexed: DupMCID = &TII->get(AArch64::DUPv4i32lane); MulMCID = &TII->get(AArch64::FMULv4f32); break; // 2X64 instructions case AArch64::FMLAv2i64_indexed: DupMCID = &TII->get(AArch64::DUPv2i64lane); MulMCID = &TII->get(AArch64::FMLAv2f64); break; case AArch64::FMLSv2i64_indexed: DupMCID = &TII->get(AArch64::DUPv2i64lane); MulMCID = &TII->get(AArch64::FMLSv2f64); break; case AArch64::FMULXv2i64_indexed: DupMCID = &TII->get(AArch64::DUPv2i64lane); MulMCID = &TII->get(AArch64::FMULXv2f64); break; case AArch64::FMULv2i64_indexed: DupMCID = &TII->get(AArch64::DUPv2i64lane); MulMCID = &TII->get(AArch64::FMULv2f64); break; // 2X32 instructions case AArch64::FMLAv2i32_indexed: RC = &AArch64::FPR64RegClass; DupMCID = &TII->get(AArch64::DUPv2i32lane); MulMCID = &TII->get(AArch64::FMLAv2f32); break; case AArch64::FMLSv2i32_indexed: RC = &AArch64::FPR64RegClass; DupMCID = &TII->get(AArch64::DUPv2i32lane); MulMCID = &TII->get(AArch64::FMLSv2f32); break; case AArch64::FMULXv2i32_indexed: RC = &AArch64::FPR64RegClass; DupMCID = &TII->get(AArch64::DUPv2i32lane); MulMCID = &TII->get(AArch64::FMULXv2f32); break; case AArch64::FMULv2i32_indexed: RC = &AArch64::FPR64RegClass; DupMCID = &TII->get(AArch64::DUPv2i32lane); MulMCID = &TII->get(AArch64::FMULv2f32); break; } SmallVector ReplInstrMCID; ReplInstrMCID.push_back(DupMCID); ReplInstrMCID.push_back(MulMCID); if (!shouldReplaceInst(MI.getParent()->getParent(), &TII->get(MI.getOpcode()), ReplInstrMCID)) return false; const DebugLoc &DL = MI.getDebugLoc(); MachineBasicBlock &MBB = *MI.getParent(); MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); // Get the operands of the current SIMD arithmetic instruction. Register MulDest = MI.getOperand(0).getReg(); Register SrcReg0 = MI.getOperand(1).getReg(); unsigned Src0IsKill = getKillRegState(MI.getOperand(1).isKill()); Register SrcReg1 = MI.getOperand(2).getReg(); unsigned Src1IsKill = getKillRegState(MI.getOperand(2).isKill()); unsigned DupDest; // Instructions of interest have either 4 or 5 operands. if (MI.getNumOperands() == 5) { Register SrcReg2 = MI.getOperand(3).getReg(); unsigned Src2IsKill = getKillRegState(MI.getOperand(3).isKill()); unsigned LaneNumber = MI.getOperand(4).getImm(); // Create a new DUP instruction. Note that if an equivalent DUP instruction // has already been created before, then use that one instead of creating // a new one. if (!reuseDUP(MI, DupMCID->getOpcode(), SrcReg2, LaneNumber, &DupDest)) { DupDest = MRI.createVirtualRegister(RC); BuildMI(MBB, MI, DL, *DupMCID, DupDest) .addReg(SrcReg2, Src2IsKill) .addImm(LaneNumber); } BuildMI(MBB, MI, DL, *MulMCID, MulDest) .addReg(SrcReg0, Src0IsKill) .addReg(SrcReg1, Src1IsKill) .addReg(DupDest, Src2IsKill); } else if (MI.getNumOperands() == 4) { unsigned LaneNumber = MI.getOperand(3).getImm(); if (!reuseDUP(MI, DupMCID->getOpcode(), SrcReg1, LaneNumber, &DupDest)) { DupDest = MRI.createVirtualRegister(RC); BuildMI(MBB, MI, DL, *DupMCID, DupDest) .addReg(SrcReg1, Src1IsKill) .addImm(LaneNumber); } BuildMI(MBB, MI, DL, *MulMCID, MulDest) .addReg(SrcReg0, Src0IsKill) .addReg(DupDest, Src1IsKill); } else { return false; } ++NumModifiedInstr; return true; } /// Load/Store Interleaving instructions are not always beneficial. /// Replace them by ZIP instructions and classical load/store. /// /// For example: /// st2 {v0.4s, v1.4s}, addr /// /// Is rewritten into: /// zip1 v2.4s, v0.4s, v1.4s /// zip2 v3.4s, v0.4s, v1.4s /// stp q2, q3, addr // /// For example: /// st4 {v0.4s, v1.4s, v2.4s, v3.4s}, addr /// /// Is rewritten into: /// zip1 v4.4s, v0.4s, v2.4s /// zip2 v5.4s, v0.4s, v2.4s /// zip1 v6.4s, v1.4s, v3.4s /// zip2 v7.4s, v1.4s, v3.4s /// zip1 v8.4s, v4.4s, v6.4s /// zip2 v9.4s, v4.4s, v6.4s /// zip1 v10.4s, v5.4s, v7.4s /// zip2 v11.4s, v5.4s, v7.4s /// stp q8, q9, addr /// stp q10, q11, addr+32 /// /// Currently only instructions related to ST2 and ST4 are considered. /// Other may be added later. /// Return true if the SIMD instruction is modified. bool AArch64SIMDInstrOpt::optimizeLdStInterleave(MachineInstr &MI) { unsigned SeqReg, AddrReg; unsigned StReg[4], StRegKill[4]; MachineInstr *DefiningMI; const DebugLoc &DL = MI.getDebugLoc(); MachineBasicBlock &MBB = *MI.getParent(); SmallVector ZipDest; SmallVector ReplInstrMCID; // If current instruction matches any of the rewriting rules, then // gather information about parameters of the new instructions. bool Match = false; for (auto &I : IRT) { if (MI.getOpcode() == I.OrigOpc) { SeqReg = MI.getOperand(0).getReg(); AddrReg = MI.getOperand(1).getReg(); DefiningMI = MRI->getUniqueVRegDef(SeqReg); unsigned NumReg = determineSrcReg(MI); if (!processSeqRegInst(DefiningMI, StReg, StRegKill, NumReg)) return false; for (auto &Repl : I.ReplOpc) { ReplInstrMCID.push_back(&TII->get(Repl)); // Generate destination registers but only for non-store instruction. if (Repl != AArch64::STPQi && Repl != AArch64::STPDi) ZipDest.push_back(MRI->createVirtualRegister(&I.RC)); } Match = true; break; } } if (!Match) return false; // Determine if it is profitable to replace MI by the series of instructions // represented in ReplInstrMCID. if (!shouldReplaceInst(MI.getParent()->getParent(), &TII->get(MI.getOpcode()), ReplInstrMCID)) return false; // Generate the replacement instructions composed of ZIP1, ZIP2, and STP (at // this point, the code generation is hardcoded and does not rely on the IRT // table used above given that code generation for ST2 replacement is somewhat // different than for ST4 replacement. We could have added more info into the // table related to how we build new instructions but we may be adding more // complexity with that). switch (MI.getOpcode()) { default: return false; case AArch64::ST2Twov16b: case AArch64::ST2Twov8b: case AArch64::ST2Twov8h: case AArch64::ST2Twov4h: case AArch64::ST2Twov4s: case AArch64::ST2Twov2s: case AArch64::ST2Twov2d: // ZIP instructions BuildMI(MBB, MI, DL, *ReplInstrMCID[0], ZipDest[0]) .addReg(StReg[0]) .addReg(StReg[1]); BuildMI(MBB, MI, DL, *ReplInstrMCID[1], ZipDest[1]) .addReg(StReg[0], StRegKill[0]) .addReg(StReg[1], StRegKill[1]); // STP instructions BuildMI(MBB, MI, DL, *ReplInstrMCID[2]) .addReg(ZipDest[0]) .addReg(ZipDest[1]) .addReg(AddrReg) .addImm(0); break; case AArch64::ST4Fourv16b: case AArch64::ST4Fourv8b: case AArch64::ST4Fourv8h: case AArch64::ST4Fourv4h: case AArch64::ST4Fourv4s: case AArch64::ST4Fourv2s: case AArch64::ST4Fourv2d: // ZIP instructions BuildMI(MBB, MI, DL, *ReplInstrMCID[0], ZipDest[0]) .addReg(StReg[0]) .addReg(StReg[2]); BuildMI(MBB, MI, DL, *ReplInstrMCID[1], ZipDest[1]) .addReg(StReg[0], StRegKill[0]) .addReg(StReg[2], StRegKill[2]); BuildMI(MBB, MI, DL, *ReplInstrMCID[2], ZipDest[2]) .addReg(StReg[1]) .addReg(StReg[3]); BuildMI(MBB, MI, DL, *ReplInstrMCID[3], ZipDest[3]) .addReg(StReg[1], StRegKill[1]) .addReg(StReg[3], StRegKill[3]); BuildMI(MBB, MI, DL, *ReplInstrMCID[4], ZipDest[4]) .addReg(ZipDest[0]) .addReg(ZipDest[2]); BuildMI(MBB, MI, DL, *ReplInstrMCID[5], ZipDest[5]) .addReg(ZipDest[0]) .addReg(ZipDest[2]); BuildMI(MBB, MI, DL, *ReplInstrMCID[6], ZipDest[6]) .addReg(ZipDest[1]) .addReg(ZipDest[3]); BuildMI(MBB, MI, DL, *ReplInstrMCID[7], ZipDest[7]) .addReg(ZipDest[1]) .addReg(ZipDest[3]); // stp instructions BuildMI(MBB, MI, DL, *ReplInstrMCID[8]) .addReg(ZipDest[4]) .addReg(ZipDest[5]) .addReg(AddrReg) .addImm(0); BuildMI(MBB, MI, DL, *ReplInstrMCID[9]) .addReg(ZipDest[6]) .addReg(ZipDest[7]) .addReg(AddrReg) .addImm(2); break; } ++NumModifiedInstr; return true; } /// Process The REG_SEQUENCE instruction, and extract the source /// operands of the ST2/4 instruction from it. /// Example of such instruction. /// %dest = REG_SEQUENCE %st2_src1, dsub0, %st2_src2, dsub1; /// Return true when the instruction is processed successfully. bool AArch64SIMDInstrOpt::processSeqRegInst(MachineInstr *DefiningMI, unsigned* StReg, unsigned* StRegKill, unsigned NumArg) const { assert(DefiningMI != nullptr); if (DefiningMI->getOpcode() != AArch64::REG_SEQUENCE) return false; for (unsigned i=0; igetOperand(2*i+1).getReg(); StRegKill[i] = getKillRegState(DefiningMI->getOperand(2*i+1).isKill()); // Validation check for the other arguments. if (DefiningMI->getOperand(2*i+2).isImm()) { switch (DefiningMI->getOperand(2*i+2).getImm()) { default: return false; case AArch64::dsub0: case AArch64::dsub1: case AArch64::dsub2: case AArch64::dsub3: case AArch64::qsub0: case AArch64::qsub1: case AArch64::qsub2: case AArch64::qsub3: break; } } else return false; } return true; } /// Return the number of useful source registers for this instruction /// (2 for ST2 and 4 for ST4). unsigned AArch64SIMDInstrOpt::determineSrcReg(MachineInstr &MI) const { switch (MI.getOpcode()) { default: llvm_unreachable("Unsupported instruction for this pass"); case AArch64::ST2Twov16b: case AArch64::ST2Twov8b: case AArch64::ST2Twov8h: case AArch64::ST2Twov4h: case AArch64::ST2Twov4s: case AArch64::ST2Twov2s: case AArch64::ST2Twov2d: return 2; case AArch64::ST4Fourv16b: case AArch64::ST4Fourv8b: case AArch64::ST4Fourv8h: case AArch64::ST4Fourv4h: case AArch64::ST4Fourv4s: case AArch64::ST4Fourv2s: case AArch64::ST4Fourv2d: return 4; } } bool AArch64SIMDInstrOpt::runOnMachineFunction(MachineFunction &MF) { if (skipFunction(MF.getFunction())) return false; TII = MF.getSubtarget().getInstrInfo(); MRI = &MF.getRegInfo(); const TargetSubtargetInfo &ST = MF.getSubtarget(); const AArch64InstrInfo *AAII = static_cast(ST.getInstrInfo()); if (!AAII) return false; SchedModel.init(&ST); if (!SchedModel.hasInstrSchedModel()) return false; bool Changed = false; for (auto OptimizationKind : {VectorElem, Interleave}) { if (!shouldExitEarly(&MF, OptimizationKind)) { SmallVector RemoveMIs; for (MachineBasicBlock &MBB : MF) { for (MachineInstr &MI : MBB) { bool InstRewrite; if (OptimizationKind == VectorElem) InstRewrite = optimizeVectElement(MI) ; else InstRewrite = optimizeLdStInterleave(MI); if (InstRewrite) { // Add MI to the list of instructions to be removed given that it // has been replaced. RemoveMIs.push_back(&MI); Changed = true; } } } for (MachineInstr *MI : RemoveMIs) MI->eraseFromParent(); } } return Changed; } /// Returns an instance of the high cost ASIMD instruction replacement /// optimization pass. FunctionPass *llvm::createAArch64SIMDInstrOptPass() { return new AArch64SIMDInstrOpt(); }