1 //===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file This file contains the AArch64 implementation of the DAG scheduling 10 /// mutation to pair instructions back to back. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "AArch64Subtarget.h" 15 #include "llvm/CodeGen/MacroFusion.h" 16 #include "llvm/CodeGen/TargetInstrInfo.h" 17 18 using namespace llvm; 19 20 namespace { 21 22 /// CMN, CMP, TST followed by Bcc 23 static bool isArithmeticBccPair(const MachineInstr *FirstMI, 24 const MachineInstr &SecondMI) { 25 if (SecondMI.getOpcode() != AArch64::Bcc) 26 return false; 27 28 // Assume the 1st instr to be a wildcard if it is unspecified. 29 if (FirstMI == nullptr) 30 return true; 31 32 switch (FirstMI->getOpcode()) { 33 case AArch64::ADDSWri: 34 case AArch64::ADDSWrr: 35 case AArch64::ADDSXri: 36 case AArch64::ADDSXrr: 37 case AArch64::ANDSWri: 38 case AArch64::ANDSWrr: 39 case AArch64::ANDSXri: 40 case AArch64::ANDSXrr: 41 case AArch64::SUBSWri: 42 case AArch64::SUBSWrr: 43 case AArch64::SUBSXri: 44 case AArch64::SUBSXrr: 45 case AArch64::BICSWrr: 46 case AArch64::BICSXrr: 47 return true; 48 case AArch64::ADDSWrs: 49 case AArch64::ADDSXrs: 50 case AArch64::ANDSWrs: 51 case AArch64::ANDSXrs: 52 case AArch64::SUBSWrs: 53 case AArch64::SUBSXrs: 54 case AArch64::BICSWrs: 55 case AArch64::BICSXrs: 56 // Shift value can be 0 making these behave like the "rr" variant... 57 return !AArch64InstrInfo::hasShiftedReg(*FirstMI); 58 } 59 60 return false; 61 } 62 63 /// ALU operations followed by CBZ/CBNZ. 64 static bool isArithmeticCbzPair(const MachineInstr *FirstMI, 65 const MachineInstr &SecondMI) { 66 if (SecondMI.getOpcode() != AArch64::CBZW && 67 SecondMI.getOpcode() != AArch64::CBZX && 68 SecondMI.getOpcode() != AArch64::CBNZW && 69 SecondMI.getOpcode() != AArch64::CBNZX) 70 return false; 71 72 // Assume the 1st instr to be a wildcard if it is unspecified. 73 if (FirstMI == nullptr) 74 return true; 75 76 switch (FirstMI->getOpcode()) { 77 case AArch64::ADDWri: 78 case AArch64::ADDWrr: 79 case AArch64::ADDXri: 80 case AArch64::ADDXrr: 81 case AArch64::ANDWri: 82 case AArch64::ANDWrr: 83 case AArch64::ANDXri: 84 case AArch64::ANDXrr: 85 case AArch64::EORWri: 86 case AArch64::EORWrr: 87 case AArch64::EORXri: 88 case AArch64::EORXrr: 89 case AArch64::ORRWri: 90 case AArch64::ORRWrr: 91 case AArch64::ORRXri: 92 case AArch64::ORRXrr: 93 case AArch64::SUBWri: 94 case AArch64::SUBWrr: 95 case AArch64::SUBXri: 96 case AArch64::SUBXrr: 97 return true; 98 case AArch64::ADDWrs: 99 case AArch64::ADDXrs: 100 case AArch64::ANDWrs: 101 case AArch64::ANDXrs: 102 case AArch64::SUBWrs: 103 case AArch64::SUBXrs: 104 case AArch64::BICWrs: 105 case AArch64::BICXrs: 106 // Shift value can be 0 making these behave like the "rr" variant... 107 return !AArch64InstrInfo::hasShiftedReg(*FirstMI); 108 } 109 110 return false; 111 } 112 113 /// AES crypto encoding or decoding. 114 static bool isAESPair(const MachineInstr *FirstMI, 115 const MachineInstr &SecondMI) { 116 // Assume the 1st instr to be a wildcard if it is unspecified. 117 switch (SecondMI.getOpcode()) { 118 // AES encode. 119 case AArch64::AESMCrr: 120 case AArch64::AESMCrrTied: 121 return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESErr; 122 // AES decode. 123 case AArch64::AESIMCrr: 124 case AArch64::AESIMCrrTied: 125 return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESDrr; 126 } 127 128 return false; 129 } 130 131 /// AESE/AESD/PMULL + EOR. 132 static bool isCryptoEORPair(const MachineInstr *FirstMI, 133 const MachineInstr &SecondMI) { 134 if (SecondMI.getOpcode() != AArch64::EORv16i8) 135 return false; 136 137 // Assume the 1st instr to be a wildcard if it is unspecified. 138 if (FirstMI == nullptr) 139 return true; 140 141 switch (FirstMI->getOpcode()) { 142 case AArch64::AESErr: 143 case AArch64::AESDrr: 144 case AArch64::PMULLv16i8: 145 case AArch64::PMULLv8i8: 146 case AArch64::PMULLv1i64: 147 case AArch64::PMULLv2i64: 148 return true; 149 } 150 151 return false; 152 } 153 154 /// Literal generation. 155 static bool isLiteralsPair(const MachineInstr *FirstMI, 156 const MachineInstr &SecondMI) { 157 // Assume the 1st instr to be a wildcard if it is unspecified. 158 159 // PC relative address. 160 if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::ADRP) && 161 SecondMI.getOpcode() == AArch64::ADDXri) 162 return true; 163 164 // 32 bit immediate. 165 if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZWi) && 166 (SecondMI.getOpcode() == AArch64::MOVKWi && 167 SecondMI.getOperand(3).getImm() == 16)) 168 return true; 169 170 // Lower half of 64 bit immediate. 171 if((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZXi) && 172 (SecondMI.getOpcode() == AArch64::MOVKXi && 173 SecondMI.getOperand(3).getImm() == 16)) 174 return true; 175 176 // Upper half of 64 bit immediate. 177 if ((FirstMI == nullptr || 178 (FirstMI->getOpcode() == AArch64::MOVKXi && 179 FirstMI->getOperand(3).getImm() == 32)) && 180 (SecondMI.getOpcode() == AArch64::MOVKXi && 181 SecondMI.getOperand(3).getImm() == 48)) 182 return true; 183 184 return false; 185 } 186 187 /// Fuse address generation and loads or stores. 188 static bool isAddressLdStPair(const MachineInstr *FirstMI, 189 const MachineInstr &SecondMI) { 190 switch (SecondMI.getOpcode()) { 191 case AArch64::STRBBui: 192 case AArch64::STRBui: 193 case AArch64::STRDui: 194 case AArch64::STRHHui: 195 case AArch64::STRHui: 196 case AArch64::STRQui: 197 case AArch64::STRSui: 198 case AArch64::STRWui: 199 case AArch64::STRXui: 200 case AArch64::LDRBBui: 201 case AArch64::LDRBui: 202 case AArch64::LDRDui: 203 case AArch64::LDRHHui: 204 case AArch64::LDRHui: 205 case AArch64::LDRQui: 206 case AArch64::LDRSui: 207 case AArch64::LDRWui: 208 case AArch64::LDRXui: 209 case AArch64::LDRSBWui: 210 case AArch64::LDRSBXui: 211 case AArch64::LDRSHWui: 212 case AArch64::LDRSHXui: 213 case AArch64::LDRSWui: 214 // Assume the 1st instr to be a wildcard if it is unspecified. 215 if (FirstMI == nullptr) 216 return true; 217 218 switch (FirstMI->getOpcode()) { 219 case AArch64::ADR: 220 return SecondMI.getOperand(2).getImm() == 0; 221 case AArch64::ADRP: 222 return true; 223 } 224 } 225 226 return false; 227 } 228 229 /// Compare and conditional select. 230 static bool isCCSelectPair(const MachineInstr *FirstMI, 231 const MachineInstr &SecondMI) { 232 // 32 bits 233 if (SecondMI.getOpcode() == AArch64::CSELWr) { 234 // Assume the 1st instr to be a wildcard if it is unspecified. 235 if (FirstMI == nullptr) 236 return true; 237 238 if (FirstMI->definesRegister(AArch64::WZR)) 239 switch (FirstMI->getOpcode()) { 240 case AArch64::SUBSWrs: 241 return !AArch64InstrInfo::hasShiftedReg(*FirstMI); 242 case AArch64::SUBSWrx: 243 return !AArch64InstrInfo::hasExtendedReg(*FirstMI); 244 case AArch64::SUBSWrr: 245 case AArch64::SUBSWri: 246 return true; 247 } 248 } 249 250 // 64 bits 251 if (SecondMI.getOpcode() == AArch64::CSELXr) { 252 // Assume the 1st instr to be a wildcard if it is unspecified. 253 if (FirstMI == nullptr) 254 return true; 255 256 if (FirstMI->definesRegister(AArch64::XZR)) 257 switch (FirstMI->getOpcode()) { 258 case AArch64::SUBSXrs: 259 return !AArch64InstrInfo::hasShiftedReg(*FirstMI); 260 case AArch64::SUBSXrx: 261 case AArch64::SUBSXrx64: 262 return !AArch64InstrInfo::hasExtendedReg(*FirstMI); 263 case AArch64::SUBSXrr: 264 case AArch64::SUBSXri: 265 return true; 266 } 267 } 268 269 return false; 270 } 271 272 // Arithmetic and logic. 273 static bool isArithmeticLogicPair(const MachineInstr *FirstMI, 274 const MachineInstr &SecondMI) { 275 if (AArch64InstrInfo::hasShiftedReg(SecondMI)) 276 return false; 277 278 switch (SecondMI.getOpcode()) { 279 // Arithmetic 280 case AArch64::ADDWrr: 281 case AArch64::ADDXrr: 282 case AArch64::SUBWrr: 283 case AArch64::SUBXrr: 284 case AArch64::ADDWrs: 285 case AArch64::ADDXrs: 286 case AArch64::SUBWrs: 287 case AArch64::SUBXrs: 288 // Logic 289 case AArch64::ANDWrr: 290 case AArch64::ANDXrr: 291 case AArch64::BICWrr: 292 case AArch64::BICXrr: 293 case AArch64::EONWrr: 294 case AArch64::EONXrr: 295 case AArch64::EORWrr: 296 case AArch64::EORXrr: 297 case AArch64::ORNWrr: 298 case AArch64::ORNXrr: 299 case AArch64::ORRWrr: 300 case AArch64::ORRXrr: 301 case AArch64::ANDWrs: 302 case AArch64::ANDXrs: 303 case AArch64::BICWrs: 304 case AArch64::BICXrs: 305 case AArch64::EONWrs: 306 case AArch64::EONXrs: 307 case AArch64::EORWrs: 308 case AArch64::EORXrs: 309 case AArch64::ORNWrs: 310 case AArch64::ORNXrs: 311 case AArch64::ORRWrs: 312 case AArch64::ORRXrs: 313 // Assume the 1st instr to be a wildcard if it is unspecified. 314 if (FirstMI == nullptr) 315 return true; 316 317 // Arithmetic 318 switch (FirstMI->getOpcode()) { 319 case AArch64::ADDWrr: 320 case AArch64::ADDXrr: 321 case AArch64::ADDSWrr: 322 case AArch64::ADDSXrr: 323 case AArch64::SUBWrr: 324 case AArch64::SUBXrr: 325 case AArch64::SUBSWrr: 326 case AArch64::SUBSXrr: 327 return true; 328 case AArch64::ADDWrs: 329 case AArch64::ADDXrs: 330 case AArch64::ADDSWrs: 331 case AArch64::ADDSXrs: 332 case AArch64::SUBWrs: 333 case AArch64::SUBXrs: 334 case AArch64::SUBSWrs: 335 case AArch64::SUBSXrs: 336 return !AArch64InstrInfo::hasShiftedReg(*FirstMI); 337 } 338 break; 339 340 // Arithmetic, setting flags. 341 case AArch64::ADDSWrr: 342 case AArch64::ADDSXrr: 343 case AArch64::SUBSWrr: 344 case AArch64::SUBSXrr: 345 case AArch64::ADDSWrs: 346 case AArch64::ADDSXrs: 347 case AArch64::SUBSWrs: 348 case AArch64::SUBSXrs: 349 // Assume the 1st instr to be a wildcard if it is unspecified. 350 if (FirstMI == nullptr) 351 return true; 352 353 // Arithmetic, not setting flags. 354 switch (FirstMI->getOpcode()) { 355 case AArch64::ADDWrr: 356 case AArch64::ADDXrr: 357 case AArch64::SUBWrr: 358 case AArch64::SUBXrr: 359 return true; 360 case AArch64::ADDWrs: 361 case AArch64::ADDXrs: 362 case AArch64::SUBWrs: 363 case AArch64::SUBXrs: 364 return !AArch64InstrInfo::hasShiftedReg(*FirstMI); 365 } 366 break; 367 } 368 369 return false; 370 } 371 372 /// \brief Check if the instr pair, FirstMI and SecondMI, should be fused 373 /// together. Given SecondMI, when FirstMI is unspecified, then check if 374 /// SecondMI may be part of a fused pair at all. 375 static bool shouldScheduleAdjacent(const TargetInstrInfo &TII, 376 const TargetSubtargetInfo &TSI, 377 const MachineInstr *FirstMI, 378 const MachineInstr &SecondMI) { 379 const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI); 380 381 // All checking functions assume that the 1st instr is a wildcard if it is 382 // unspecified. 383 if (ST.hasArithmeticBccFusion() && isArithmeticBccPair(FirstMI, SecondMI)) 384 return true; 385 if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI)) 386 return true; 387 if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI)) 388 return true; 389 if (ST.hasFuseCryptoEOR() && isCryptoEORPair(FirstMI, SecondMI)) 390 return true; 391 if (ST.hasFuseLiterals() && isLiteralsPair(FirstMI, SecondMI)) 392 return true; 393 if (ST.hasFuseAddress() && isAddressLdStPair(FirstMI, SecondMI)) 394 return true; 395 if (ST.hasFuseCCSelect() && isCCSelectPair(FirstMI, SecondMI)) 396 return true; 397 if (ST.hasFuseArithmeticLogic() && isArithmeticLogicPair(FirstMI, SecondMI)) 398 return true; 399 400 return false; 401 } 402 403 } // end namespace 404 405 406 namespace llvm { 407 408 std::unique_ptr<ScheduleDAGMutation> createAArch64MacroFusionDAGMutation () { 409 return createMacroFusionDAGMutation(shouldScheduleAdjacent); 410 } 411 412 } // end namespace llvm 413