1 // 2 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 3 // See https://llvm.org/LICENSE.txt for license information. 4 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 5 // 6 //===----------------------------------------------------------------------===// 7 // 8 // This file contains a pass that performs optimization on SIMD instructions 9 // with high latency by splitting them into more efficient series of 10 // instructions. 11 // 12 // 1. Rewrite certain SIMD instructions with vector element due to their 13 // inefficiency on some targets. 14 // 15 // For example: 16 // fmla v0.4s, v1.4s, v2.s[1] 17 // 18 // Is rewritten into: 19 // dup v3.4s, v2.s[1] 20 // fmla v0.4s, v1.4s, v3.4s 21 // 22 // 2. Rewrite interleaved memory access instructions due to their 23 // inefficiency on some targets. 24 // 25 // For example: 26 // st2 {v0.4s, v1.4s}, addr 27 // 28 // Is rewritten into: 29 // zip1 v2.4s, v0.4s, v1.4s 30 // zip2 v3.4s, v0.4s, v1.4s 31 // stp q2, q3, addr 32 // 33 //===----------------------------------------------------------------------===// 34 35 #include "AArch64InstrInfo.h" 36 #include "llvm/ADT/SmallVector.h" 37 #include "llvm/ADT/Statistic.h" 38 #include "llvm/ADT/StringRef.h" 39 #include "llvm/CodeGen/MachineBasicBlock.h" 40 #include "llvm/CodeGen/MachineFunction.h" 41 #include "llvm/CodeGen/MachineFunctionPass.h" 42 #include "llvm/CodeGen/MachineInstr.h" 43 #include "llvm/CodeGen/MachineInstrBuilder.h" 44 #include "llvm/CodeGen/MachineOperand.h" 45 #include "llvm/CodeGen/MachineRegisterInfo.h" 46 #include "llvm/CodeGen/TargetInstrInfo.h" 47 #include "llvm/CodeGen/TargetSchedule.h" 48 #include "llvm/CodeGen/TargetSubtargetInfo.h" 49 #include "llvm/MC/MCInstrDesc.h" 50 #include "llvm/MC/MCSchedule.h" 51 #include "llvm/Pass.h" 52 #include <unordered_map> 53 54 using namespace llvm; 55 56 #define DEBUG_TYPE "aarch64-simdinstr-opt" 57 58 STATISTIC(NumModifiedInstr, 59 "Number of SIMD instructions modified"); 60 61 #define AARCH64_VECTOR_BY_ELEMENT_OPT_NAME \ 62 "AArch64 SIMD instructions optimization pass" 63 64 namespace { 65 66 struct AArch64SIMDInstrOpt : public MachineFunctionPass { 67 static char ID; 68 69 const TargetInstrInfo *TII; 70 MachineRegisterInfo *MRI; 71 TargetSchedModel SchedModel; 72 73 // The two maps below are used to cache decisions instead of recomputing: 74 // This is used to cache instruction replacement decisions within function 75 // units and across function units. 76 std::map<std::pair<unsigned, std::string>, bool> SIMDInstrTable; 77 // This is used to cache the decision of whether to leave the interleaved 78 // store instructions replacement pass early or not for a particular target. 79 std::unordered_map<std::string, bool> InterlEarlyExit; 80 81 typedef enum { 82 VectorElem, 83 Interleave 84 } Subpass; 85 86 // Instruction represented by OrigOpc is replaced by instructions in ReplOpc. 87 struct InstReplInfo { 88 unsigned OrigOpc; 89 std::vector<unsigned> ReplOpc; 90 const TargetRegisterClass RC; 91 }; 92 93 #define RuleST2(OpcOrg, OpcR0, OpcR1, OpcR2, RC) \ 94 {OpcOrg, {OpcR0, OpcR1, OpcR2}, RC} 95 #define RuleST4(OpcOrg, OpcR0, OpcR1, OpcR2, OpcR3, OpcR4, OpcR5, OpcR6, \ 96 OpcR7, OpcR8, OpcR9, RC) \ 97 {OpcOrg, \ 98 {OpcR0, OpcR1, OpcR2, OpcR3, OpcR4, OpcR5, OpcR6, OpcR7, OpcR8, OpcR9}, RC} 99 100 // The Instruction Replacement Table: 101 std::vector<InstReplInfo> IRT = { 102 // ST2 instructions 103 RuleST2(AArch64::ST2Twov2d, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, 104 AArch64::STPQi, AArch64::FPR128RegClass), 105 RuleST2(AArch64::ST2Twov4s, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, 106 AArch64::STPQi, AArch64::FPR128RegClass), 107 RuleST2(AArch64::ST2Twov2s, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, 108 AArch64::STPDi, AArch64::FPR64RegClass), 109 RuleST2(AArch64::ST2Twov8h, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, 110 AArch64::STPQi, AArch64::FPR128RegClass), 111 RuleST2(AArch64::ST2Twov4h, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, 112 AArch64::STPDi, AArch64::FPR64RegClass), 113 RuleST2(AArch64::ST2Twov16b, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, 114 AArch64::STPQi, AArch64::FPR128RegClass), 115 RuleST2(AArch64::ST2Twov8b, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, 116 AArch64::STPDi, AArch64::FPR64RegClass), 117 // ST4 instructions 118 RuleST4(AArch64::ST4Fourv2d, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, 119 AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, AArch64::ZIP1v2i64, 120 AArch64::ZIP2v2i64, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, 121 AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass), 122 RuleST4(AArch64::ST4Fourv4s, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, 123 AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, AArch64::ZIP1v4i32, 124 AArch64::ZIP2v4i32, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, 125 AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass), 126 RuleST4(AArch64::ST4Fourv2s, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, 127 AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, AArch64::ZIP1v2i32, 128 AArch64::ZIP2v2i32, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, 129 AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass), 130 RuleST4(AArch64::ST4Fourv8h, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, 131 AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, AArch64::ZIP1v8i16, 132 AArch64::ZIP2v8i16, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, 133 AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass), 134 RuleST4(AArch64::ST4Fourv4h, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, 135 AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, AArch64::ZIP1v4i16, 136 AArch64::ZIP2v4i16, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, 137 AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass), 138 RuleST4(AArch64::ST4Fourv16b, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, 139 AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, AArch64::ZIP1v16i8, 140 AArch64::ZIP2v16i8, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, 141 AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass), 142 RuleST4(AArch64::ST4Fourv8b, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, 143 AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, AArch64::ZIP1v8i8, 144 AArch64::ZIP2v8i8, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, 145 AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass) 146 }; 147 148 // A costly instruction is replaced in this work by N efficient instructions 149 // The maximum of N is curently 10 and it is for ST4 case. 150 static const unsigned MaxNumRepl = 10; 151 152 AArch64SIMDInstrOpt() : MachineFunctionPass(ID) { 153 initializeAArch64SIMDInstrOptPass(*PassRegistry::getPassRegistry()); 154 } 155 156 /// Based only on latency of instructions, determine if it is cost efficient 157 /// to replace the instruction InstDesc by the instructions stored in the 158 /// array InstDescRepl. 159 /// Return true if replacement is expected to be faster. 160 bool shouldReplaceInst(MachineFunction *MF, const MCInstrDesc *InstDesc, 161 SmallVectorImpl<const MCInstrDesc*> &ReplInstrMCID); 162 163 /// Determine if we need to exit the instruction replacement optimization 164 /// passes early. This makes sure that no compile time is spent in this pass 165 /// for targets with no need for any of these optimizations. 166 /// Return true if early exit of the pass is recommended. 167 bool shouldExitEarly(MachineFunction *MF, Subpass SP); 168 169 /// Check whether an equivalent DUP instruction has already been 170 /// created or not. 171 /// Return true when the DUP instruction already exists. In this case, 172 /// DestReg will point to the destination of the already created DUP. 173 bool reuseDUP(MachineInstr &MI, unsigned DupOpcode, unsigned SrcReg, 174 unsigned LaneNumber, unsigned *DestReg) const; 175 176 /// Certain SIMD instructions with vector element operand are not efficient. 177 /// Rewrite them into SIMD instructions with vector operands. This rewrite 178 /// is driven by the latency of the instructions. 179 /// Return true if the SIMD instruction is modified. 180 bool optimizeVectElement(MachineInstr &MI); 181 182 /// Process The REG_SEQUENCE instruction, and extract the source 183 /// operands of the ST2/4 instruction from it. 184 /// Example of such instructions. 185 /// %dest = REG_SEQUENCE %st2_src1, dsub0, %st2_src2, dsub1; 186 /// Return true when the instruction is processed successfully. 187 bool processSeqRegInst(MachineInstr *DefiningMI, unsigned* StReg, 188 unsigned* StRegKill, unsigned NumArg) const; 189 190 /// Load/Store Interleaving instructions are not always beneficial. 191 /// Replace them by ZIP instructionand classical load/store. 192 /// Return true if the SIMD instruction is modified. 193 bool optimizeLdStInterleave(MachineInstr &MI); 194 195 /// Return the number of useful source registers for this 196 /// instruction (2 for ST2 and 4 for ST4). 197 unsigned determineSrcReg(MachineInstr &MI) const; 198 199 bool runOnMachineFunction(MachineFunction &Fn) override; 200 201 StringRef getPassName() const override { 202 return AARCH64_VECTOR_BY_ELEMENT_OPT_NAME; 203 } 204 }; 205 206 char AArch64SIMDInstrOpt::ID = 0; 207 208 } // end anonymous namespace 209 210 INITIALIZE_PASS(AArch64SIMDInstrOpt, "aarch64-simdinstr-opt", 211 AARCH64_VECTOR_BY_ELEMENT_OPT_NAME, false, false) 212 213 /// Based only on latency of instructions, determine if it is cost efficient 214 /// to replace the instruction InstDesc by the instructions stored in the 215 /// array InstDescRepl. 216 /// Return true if replacement is expected to be faster. 217 bool AArch64SIMDInstrOpt:: 218 shouldReplaceInst(MachineFunction *MF, const MCInstrDesc *InstDesc, 219 SmallVectorImpl<const MCInstrDesc*> &InstDescRepl) { 220 // Check if replacement decision is already available in the cached table. 221 // if so, return it. 222 std::string Subtarget = std::string(SchedModel.getSubtargetInfo()->getCPU()); 223 auto InstID = std::make_pair(InstDesc->getOpcode(), Subtarget); 224 auto It = SIMDInstrTable.find(InstID); 225 if (It != SIMDInstrTable.end()) 226 return It->second; 227 228 unsigned SCIdx = InstDesc->getSchedClass(); 229 const MCSchedClassDesc *SCDesc = 230 SchedModel.getMCSchedModel()->getSchedClassDesc(SCIdx); 231 232 // If a target does not define resources for the instructions 233 // of interest, then return false for no replacement. 234 const MCSchedClassDesc *SCDescRepl; 235 if (!SCDesc->isValid() || SCDesc->isVariant()) 236 { 237 SIMDInstrTable[InstID] = false; 238 return false; 239 } 240 for (auto IDesc : InstDescRepl) 241 { 242 SCDescRepl = SchedModel.getMCSchedModel()->getSchedClassDesc( 243 IDesc->getSchedClass()); 244 if (!SCDescRepl->isValid() || SCDescRepl->isVariant()) 245 { 246 SIMDInstrTable[InstID] = false; 247 return false; 248 } 249 } 250 251 // Replacement cost. 252 unsigned ReplCost = 0; 253 for (auto IDesc :InstDescRepl) 254 ReplCost += SchedModel.computeInstrLatency(IDesc->getOpcode()); 255 256 if (SchedModel.computeInstrLatency(InstDesc->getOpcode()) > ReplCost) 257 { 258 SIMDInstrTable[InstID] = true; 259 return true; 260 } 261 else 262 { 263 SIMDInstrTable[InstID] = false; 264 return false; 265 } 266 } 267 268 /// Determine if we need to exit this pass for a kind of instruction replacement 269 /// early. This makes sure that no compile time is spent in this pass for 270 /// targets with no need for any of these optimizations beyond performing this 271 /// check. 272 /// Return true if early exit of this pass for a kind of instruction 273 /// replacement is recommended for a target. 274 bool AArch64SIMDInstrOpt::shouldExitEarly(MachineFunction *MF, Subpass SP) { 275 const MCInstrDesc* OriginalMCID; 276 SmallVector<const MCInstrDesc*, MaxNumRepl> ReplInstrMCID; 277 278 switch (SP) { 279 // For this optimization, check by comparing the latency of a representative 280 // instruction to that of the replacement instructions. 281 // TODO: check for all concerned instructions. 282 case VectorElem: 283 OriginalMCID = &TII->get(AArch64::FMLAv4i32_indexed); 284 ReplInstrMCID.push_back(&TII->get(AArch64::DUPv4i32lane)); 285 ReplInstrMCID.push_back(&TII->get(AArch64::FMLAv4f32)); 286 if (shouldReplaceInst(MF, OriginalMCID, ReplInstrMCID)) 287 return false; 288 break; 289 290 // For this optimization, check for all concerned instructions. 291 case Interleave: 292 std::string Subtarget = 293 std::string(SchedModel.getSubtargetInfo()->getCPU()); 294 auto It = InterlEarlyExit.find(Subtarget); 295 if (It != InterlEarlyExit.end()) 296 return It->second; 297 298 for (auto &I : IRT) { 299 OriginalMCID = &TII->get(I.OrigOpc); 300 for (auto &Repl : I.ReplOpc) 301 ReplInstrMCID.push_back(&TII->get(Repl)); 302 if (shouldReplaceInst(MF, OriginalMCID, ReplInstrMCID)) { 303 InterlEarlyExit[Subtarget] = false; 304 return false; 305 } 306 ReplInstrMCID.clear(); 307 } 308 InterlEarlyExit[Subtarget] = true; 309 break; 310 } 311 312 return true; 313 } 314 315 /// Check whether an equivalent DUP instruction has already been 316 /// created or not. 317 /// Return true when the DUP instruction already exists. In this case, 318 /// DestReg will point to the destination of the already created DUP. 319 bool AArch64SIMDInstrOpt::reuseDUP(MachineInstr &MI, unsigned DupOpcode, 320 unsigned SrcReg, unsigned LaneNumber, 321 unsigned *DestReg) const { 322 for (MachineBasicBlock::iterator MII = MI, MIE = MI.getParent()->begin(); 323 MII != MIE;) { 324 MII--; 325 MachineInstr *CurrentMI = &*MII; 326 327 if (CurrentMI->getOpcode() == DupOpcode && 328 CurrentMI->getNumOperands() == 3 && 329 CurrentMI->getOperand(1).getReg() == SrcReg && 330 CurrentMI->getOperand(2).getImm() == LaneNumber) { 331 *DestReg = CurrentMI->getOperand(0).getReg(); 332 return true; 333 } 334 } 335 336 return false; 337 } 338 339 /// Certain SIMD instructions with vector element operand are not efficient. 340 /// Rewrite them into SIMD instructions with vector operands. This rewrite 341 /// is driven by the latency of the instructions. 342 /// The instruction of concerns are for the time being FMLA, FMLS, FMUL, 343 /// and FMULX and hence they are hardcoded. 344 /// 345 /// For example: 346 /// fmla v0.4s, v1.4s, v2.s[1] 347 /// 348 /// Is rewritten into 349 /// dup v3.4s, v2.s[1] // DUP not necessary if redundant 350 /// fmla v0.4s, v1.4s, v3.4s 351 /// 352 /// Return true if the SIMD instruction is modified. 353 bool AArch64SIMDInstrOpt::optimizeVectElement(MachineInstr &MI) { 354 const MCInstrDesc *MulMCID, *DupMCID; 355 const TargetRegisterClass *RC = &AArch64::FPR128RegClass; 356 357 switch (MI.getOpcode()) { 358 default: 359 return false; 360 361 // 4X32 instructions 362 case AArch64::FMLAv4i32_indexed: 363 DupMCID = &TII->get(AArch64::DUPv4i32lane); 364 MulMCID = &TII->get(AArch64::FMLAv4f32); 365 break; 366 case AArch64::FMLSv4i32_indexed: 367 DupMCID = &TII->get(AArch64::DUPv4i32lane); 368 MulMCID = &TII->get(AArch64::FMLSv4f32); 369 break; 370 case AArch64::FMULXv4i32_indexed: 371 DupMCID = &TII->get(AArch64::DUPv4i32lane); 372 MulMCID = &TII->get(AArch64::FMULXv4f32); 373 break; 374 case AArch64::FMULv4i32_indexed: 375 DupMCID = &TII->get(AArch64::DUPv4i32lane); 376 MulMCID = &TII->get(AArch64::FMULv4f32); 377 break; 378 379 // 2X64 instructions 380 case AArch64::FMLAv2i64_indexed: 381 DupMCID = &TII->get(AArch64::DUPv2i64lane); 382 MulMCID = &TII->get(AArch64::FMLAv2f64); 383 break; 384 case AArch64::FMLSv2i64_indexed: 385 DupMCID = &TII->get(AArch64::DUPv2i64lane); 386 MulMCID = &TII->get(AArch64::FMLSv2f64); 387 break; 388 case AArch64::FMULXv2i64_indexed: 389 DupMCID = &TII->get(AArch64::DUPv2i64lane); 390 MulMCID = &TII->get(AArch64::FMULXv2f64); 391 break; 392 case AArch64::FMULv2i64_indexed: 393 DupMCID = &TII->get(AArch64::DUPv2i64lane); 394 MulMCID = &TII->get(AArch64::FMULv2f64); 395 break; 396 397 // 2X32 instructions 398 case AArch64::FMLAv2i32_indexed: 399 RC = &AArch64::FPR64RegClass; 400 DupMCID = &TII->get(AArch64::DUPv2i32lane); 401 MulMCID = &TII->get(AArch64::FMLAv2f32); 402 break; 403 case AArch64::FMLSv2i32_indexed: 404 RC = &AArch64::FPR64RegClass; 405 DupMCID = &TII->get(AArch64::DUPv2i32lane); 406 MulMCID = &TII->get(AArch64::FMLSv2f32); 407 break; 408 case AArch64::FMULXv2i32_indexed: 409 RC = &AArch64::FPR64RegClass; 410 DupMCID = &TII->get(AArch64::DUPv2i32lane); 411 MulMCID = &TII->get(AArch64::FMULXv2f32); 412 break; 413 case AArch64::FMULv2i32_indexed: 414 RC = &AArch64::FPR64RegClass; 415 DupMCID = &TII->get(AArch64::DUPv2i32lane); 416 MulMCID = &TII->get(AArch64::FMULv2f32); 417 break; 418 } 419 420 SmallVector<const MCInstrDesc*, 2> ReplInstrMCID; 421 ReplInstrMCID.push_back(DupMCID); 422 ReplInstrMCID.push_back(MulMCID); 423 if (!shouldReplaceInst(MI.getParent()->getParent(), &TII->get(MI.getOpcode()), 424 ReplInstrMCID)) 425 return false; 426 427 const DebugLoc &DL = MI.getDebugLoc(); 428 MachineBasicBlock &MBB = *MI.getParent(); 429 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 430 431 // Get the operands of the current SIMD arithmetic instruction. 432 Register MulDest = MI.getOperand(0).getReg(); 433 Register SrcReg0 = MI.getOperand(1).getReg(); 434 unsigned Src0IsKill = getKillRegState(MI.getOperand(1).isKill()); 435 Register SrcReg1 = MI.getOperand(2).getReg(); 436 unsigned Src1IsKill = getKillRegState(MI.getOperand(2).isKill()); 437 unsigned DupDest; 438 439 // Instructions of interest have either 4 or 5 operands. 440 if (MI.getNumOperands() == 5) { 441 Register SrcReg2 = MI.getOperand(3).getReg(); 442 unsigned Src2IsKill = getKillRegState(MI.getOperand(3).isKill()); 443 unsigned LaneNumber = MI.getOperand(4).getImm(); 444 // Create a new DUP instruction. Note that if an equivalent DUP instruction 445 // has already been created before, then use that one instead of creating 446 // a new one. 447 if (!reuseDUP(MI, DupMCID->getOpcode(), SrcReg2, LaneNumber, &DupDest)) { 448 DupDest = MRI.createVirtualRegister(RC); 449 BuildMI(MBB, MI, DL, *DupMCID, DupDest) 450 .addReg(SrcReg2, Src2IsKill) 451 .addImm(LaneNumber); 452 } 453 BuildMI(MBB, MI, DL, *MulMCID, MulDest) 454 .addReg(SrcReg0, Src0IsKill) 455 .addReg(SrcReg1, Src1IsKill) 456 .addReg(DupDest, Src2IsKill); 457 } else if (MI.getNumOperands() == 4) { 458 unsigned LaneNumber = MI.getOperand(3).getImm(); 459 if (!reuseDUP(MI, DupMCID->getOpcode(), SrcReg1, LaneNumber, &DupDest)) { 460 DupDest = MRI.createVirtualRegister(RC); 461 BuildMI(MBB, MI, DL, *DupMCID, DupDest) 462 .addReg(SrcReg1, Src1IsKill) 463 .addImm(LaneNumber); 464 } 465 BuildMI(MBB, MI, DL, *MulMCID, MulDest) 466 .addReg(SrcReg0, Src0IsKill) 467 .addReg(DupDest, Src1IsKill); 468 } else { 469 return false; 470 } 471 472 ++NumModifiedInstr; 473 return true; 474 } 475 476 /// Load/Store Interleaving instructions are not always beneficial. 477 /// Replace them by ZIP instructions and classical load/store. 478 /// 479 /// For example: 480 /// st2 {v0.4s, v1.4s}, addr 481 /// 482 /// Is rewritten into: 483 /// zip1 v2.4s, v0.4s, v1.4s 484 /// zip2 v3.4s, v0.4s, v1.4s 485 /// stp q2, q3, addr 486 // 487 /// For example: 488 /// st4 {v0.4s, v1.4s, v2.4s, v3.4s}, addr 489 /// 490 /// Is rewritten into: 491 /// zip1 v4.4s, v0.4s, v2.4s 492 /// zip2 v5.4s, v0.4s, v2.4s 493 /// zip1 v6.4s, v1.4s, v3.4s 494 /// zip2 v7.4s, v1.4s, v3.4s 495 /// zip1 v8.4s, v4.4s, v6.4s 496 /// zip2 v9.4s, v4.4s, v6.4s 497 /// zip1 v10.4s, v5.4s, v7.4s 498 /// zip2 v11.4s, v5.4s, v7.4s 499 /// stp q8, q9, addr 500 /// stp q10, q11, addr+32 501 /// 502 /// Currently only instructions related to ST2 and ST4 are considered. 503 /// Other may be added later. 504 /// Return true if the SIMD instruction is modified. 505 bool AArch64SIMDInstrOpt::optimizeLdStInterleave(MachineInstr &MI) { 506 507 unsigned SeqReg, AddrReg; 508 unsigned StReg[4], StRegKill[4]; 509 MachineInstr *DefiningMI; 510 const DebugLoc &DL = MI.getDebugLoc(); 511 MachineBasicBlock &MBB = *MI.getParent(); 512 SmallVector<unsigned, MaxNumRepl> ZipDest; 513 SmallVector<const MCInstrDesc*, MaxNumRepl> ReplInstrMCID; 514 515 // If current instruction matches any of the rewriting rules, then 516 // gather information about parameters of the new instructions. 517 bool Match = false; 518 for (auto &I : IRT) { 519 if (MI.getOpcode() == I.OrigOpc) { 520 SeqReg = MI.getOperand(0).getReg(); 521 AddrReg = MI.getOperand(1).getReg(); 522 DefiningMI = MRI->getUniqueVRegDef(SeqReg); 523 unsigned NumReg = determineSrcReg(MI); 524 if (!processSeqRegInst(DefiningMI, StReg, StRegKill, NumReg)) 525 return false; 526 527 for (auto &Repl : I.ReplOpc) { 528 ReplInstrMCID.push_back(&TII->get(Repl)); 529 // Generate destination registers but only for non-store instruction. 530 if (Repl != AArch64::STPQi && Repl != AArch64::STPDi) 531 ZipDest.push_back(MRI->createVirtualRegister(&I.RC)); 532 } 533 Match = true; 534 break; 535 } 536 } 537 538 if (!Match) 539 return false; 540 541 // Determine if it is profitable to replace MI by the series of instructions 542 // represented in ReplInstrMCID. 543 if (!shouldReplaceInst(MI.getParent()->getParent(), &TII->get(MI.getOpcode()), 544 ReplInstrMCID)) 545 return false; 546 547 // Generate the replacement instructions composed of ZIP1, ZIP2, and STP (at 548 // this point, the code generation is hardcoded and does not rely on the IRT 549 // table used above given that code generation for ST2 replacement is somewhat 550 // different than for ST4 replacement. We could have added more info into the 551 // table related to how we build new instructions but we may be adding more 552 // complexity with that). 553 switch (MI.getOpcode()) { 554 default: 555 return false; 556 557 case AArch64::ST2Twov16b: 558 case AArch64::ST2Twov8b: 559 case AArch64::ST2Twov8h: 560 case AArch64::ST2Twov4h: 561 case AArch64::ST2Twov4s: 562 case AArch64::ST2Twov2s: 563 case AArch64::ST2Twov2d: 564 // ZIP instructions 565 BuildMI(MBB, MI, DL, *ReplInstrMCID[0], ZipDest[0]) 566 .addReg(StReg[0]) 567 .addReg(StReg[1]); 568 BuildMI(MBB, MI, DL, *ReplInstrMCID[1], ZipDest[1]) 569 .addReg(StReg[0], StRegKill[0]) 570 .addReg(StReg[1], StRegKill[1]); 571 // STP instructions 572 BuildMI(MBB, MI, DL, *ReplInstrMCID[2]) 573 .addReg(ZipDest[0]) 574 .addReg(ZipDest[1]) 575 .addReg(AddrReg) 576 .addImm(0); 577 break; 578 579 case AArch64::ST4Fourv16b: 580 case AArch64::ST4Fourv8b: 581 case AArch64::ST4Fourv8h: 582 case AArch64::ST4Fourv4h: 583 case AArch64::ST4Fourv4s: 584 case AArch64::ST4Fourv2s: 585 case AArch64::ST4Fourv2d: 586 // ZIP instructions 587 BuildMI(MBB, MI, DL, *ReplInstrMCID[0], ZipDest[0]) 588 .addReg(StReg[0]) 589 .addReg(StReg[2]); 590 BuildMI(MBB, MI, DL, *ReplInstrMCID[1], ZipDest[1]) 591 .addReg(StReg[0], StRegKill[0]) 592 .addReg(StReg[2], StRegKill[2]); 593 BuildMI(MBB, MI, DL, *ReplInstrMCID[2], ZipDest[2]) 594 .addReg(StReg[1]) 595 .addReg(StReg[3]); 596 BuildMI(MBB, MI, DL, *ReplInstrMCID[3], ZipDest[3]) 597 .addReg(StReg[1], StRegKill[1]) 598 .addReg(StReg[3], StRegKill[3]); 599 BuildMI(MBB, MI, DL, *ReplInstrMCID[4], ZipDest[4]) 600 .addReg(ZipDest[0]) 601 .addReg(ZipDest[2]); 602 BuildMI(MBB, MI, DL, *ReplInstrMCID[5], ZipDest[5]) 603 .addReg(ZipDest[0]) 604 .addReg(ZipDest[2]); 605 BuildMI(MBB, MI, DL, *ReplInstrMCID[6], ZipDest[6]) 606 .addReg(ZipDest[1]) 607 .addReg(ZipDest[3]); 608 BuildMI(MBB, MI, DL, *ReplInstrMCID[7], ZipDest[7]) 609 .addReg(ZipDest[1]) 610 .addReg(ZipDest[3]); 611 // stp instructions 612 BuildMI(MBB, MI, DL, *ReplInstrMCID[8]) 613 .addReg(ZipDest[4]) 614 .addReg(ZipDest[5]) 615 .addReg(AddrReg) 616 .addImm(0); 617 BuildMI(MBB, MI, DL, *ReplInstrMCID[9]) 618 .addReg(ZipDest[6]) 619 .addReg(ZipDest[7]) 620 .addReg(AddrReg) 621 .addImm(2); 622 break; 623 } 624 625 ++NumModifiedInstr; 626 return true; 627 } 628 629 /// Process The REG_SEQUENCE instruction, and extract the source 630 /// operands of the ST2/4 instruction from it. 631 /// Example of such instruction. 632 /// %dest = REG_SEQUENCE %st2_src1, dsub0, %st2_src2, dsub1; 633 /// Return true when the instruction is processed successfully. 634 bool AArch64SIMDInstrOpt::processSeqRegInst(MachineInstr *DefiningMI, 635 unsigned* StReg, unsigned* StRegKill, unsigned NumArg) const { 636 assert (DefiningMI != NULL); 637 if (DefiningMI->getOpcode() != AArch64::REG_SEQUENCE) 638 return false; 639 640 for (unsigned i=0; i<NumArg; i++) { 641 StReg[i] = DefiningMI->getOperand(2*i+1).getReg(); 642 StRegKill[i] = getKillRegState(DefiningMI->getOperand(2*i+1).isKill()); 643 644 // Sanity check for the other arguments. 645 if (DefiningMI->getOperand(2*i+2).isImm()) { 646 switch (DefiningMI->getOperand(2*i+2).getImm()) { 647 default: 648 return false; 649 650 case AArch64::dsub0: 651 case AArch64::dsub1: 652 case AArch64::dsub2: 653 case AArch64::dsub3: 654 case AArch64::qsub0: 655 case AArch64::qsub1: 656 case AArch64::qsub2: 657 case AArch64::qsub3: 658 break; 659 } 660 } 661 else 662 return false; 663 } 664 return true; 665 } 666 667 /// Return the number of useful source registers for this instruction 668 /// (2 for ST2 and 4 for ST4). 669 unsigned AArch64SIMDInstrOpt::determineSrcReg(MachineInstr &MI) const { 670 switch (MI.getOpcode()) { 671 default: 672 llvm_unreachable("Unsupported instruction for this pass"); 673 674 case AArch64::ST2Twov16b: 675 case AArch64::ST2Twov8b: 676 case AArch64::ST2Twov8h: 677 case AArch64::ST2Twov4h: 678 case AArch64::ST2Twov4s: 679 case AArch64::ST2Twov2s: 680 case AArch64::ST2Twov2d: 681 return 2; 682 683 case AArch64::ST4Fourv16b: 684 case AArch64::ST4Fourv8b: 685 case AArch64::ST4Fourv8h: 686 case AArch64::ST4Fourv4h: 687 case AArch64::ST4Fourv4s: 688 case AArch64::ST4Fourv2s: 689 case AArch64::ST4Fourv2d: 690 return 4; 691 } 692 } 693 694 bool AArch64SIMDInstrOpt::runOnMachineFunction(MachineFunction &MF) { 695 if (skipFunction(MF.getFunction())) 696 return false; 697 698 TII = MF.getSubtarget().getInstrInfo(); 699 MRI = &MF.getRegInfo(); 700 const TargetSubtargetInfo &ST = MF.getSubtarget(); 701 const AArch64InstrInfo *AAII = 702 static_cast<const AArch64InstrInfo *>(ST.getInstrInfo()); 703 if (!AAII) 704 return false; 705 SchedModel.init(&ST); 706 if (!SchedModel.hasInstrSchedModel()) 707 return false; 708 709 bool Changed = false; 710 for (auto OptimizationKind : {VectorElem, Interleave}) { 711 if (!shouldExitEarly(&MF, OptimizationKind)) { 712 SmallVector<MachineInstr *, 8> RemoveMIs; 713 for (MachineBasicBlock &MBB : MF) { 714 for (MachineBasicBlock::iterator MII = MBB.begin(), MIE = MBB.end(); 715 MII != MIE;) { 716 MachineInstr &MI = *MII; 717 bool InstRewrite; 718 if (OptimizationKind == VectorElem) 719 InstRewrite = optimizeVectElement(MI) ; 720 else 721 InstRewrite = optimizeLdStInterleave(MI); 722 if (InstRewrite) { 723 // Add MI to the list of instructions to be removed given that it 724 // has been replaced. 725 RemoveMIs.push_back(&MI); 726 Changed = true; 727 } 728 ++MII; 729 } 730 } 731 for (MachineInstr *MI : RemoveMIs) 732 MI->eraseFromParent(); 733 } 734 } 735 736 return Changed; 737 } 738 739 /// Returns an instance of the high cost ASIMD instruction replacement 740 /// optimization pass. 741 FunctionPass *llvm::createAArch64SIMDInstrOptPass() { 742 return new AArch64SIMDInstrOpt(); 743 } 744