1 //===- utils/TableGen/X86FoldTablesEmitter.cpp - X86 backend-*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This tablegen backend is responsible for emitting the memory fold tables of 10 // the X86 backend instructions. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "CodeGenTarget.h" 15 #include "X86RecognizableInstr.h" 16 #include "llvm/Support/FormattedStream.h" 17 #include "llvm/TableGen/Error.h" 18 #include "llvm/TableGen/TableGenBackend.h" 19 20 using namespace llvm; 21 22 namespace { 23 24 // 3 possible strategies for the unfolding flag (TB_NO_REVERSE) of the 25 // manual added entries. 26 enum UnfoldStrategy { 27 UNFOLD, // Allow unfolding 28 NO_UNFOLD, // Prevent unfolding 29 NO_STRATEGY // Make decision according to operands' sizes 30 }; 31 32 // Represents an entry in the manual mapped instructions set. 33 struct ManualMapEntry { 34 const char *RegInstStr; 35 const char *MemInstStr; 36 UnfoldStrategy Strategy; 37 38 ManualMapEntry(const char *RegInstStr, const char *MemInstStr, 39 UnfoldStrategy Strategy = NO_STRATEGY) 40 : RegInstStr(RegInstStr), MemInstStr(MemInstStr), Strategy(Strategy) {} 41 }; 42 43 // List of instructions requiring explicitly aligned memory. 44 const char *ExplicitAlign[] = {"MOVDQA", "MOVAPS", "MOVAPD", "MOVNTPS", 45 "MOVNTPD", "MOVNTDQ", "MOVNTDQA"}; 46 47 // List of instructions NOT requiring explicit memory alignment. 48 const char *ExplicitUnalign[] = {"MOVDQU", "MOVUPS", "MOVUPD", 49 "PCMPESTRM", "PCMPESTRI", 50 "PCMPISTRM", "PCMPISTRI" }; 51 52 // For manually mapping instructions that do not match by their encoding. 53 const ManualMapEntry ManualMapSet[] = { 54 { "ADD16ri_DB", "ADD16mi", NO_UNFOLD }, 55 { "ADD16ri8_DB", "ADD16mi8", NO_UNFOLD }, 56 { "ADD16rr_DB", "ADD16mr", NO_UNFOLD }, 57 { "ADD32ri_DB", "ADD32mi", NO_UNFOLD }, 58 { "ADD32ri8_DB", "ADD32mi8", NO_UNFOLD }, 59 { "ADD32rr_DB", "ADD32mr", NO_UNFOLD }, 60 { "ADD64ri32_DB", "ADD64mi32", NO_UNFOLD }, 61 { "ADD64ri8_DB", "ADD64mi8", NO_UNFOLD }, 62 { "ADD64rr_DB", "ADD64mr", NO_UNFOLD }, 63 { "ADD8ri_DB", "ADD8mi", NO_UNFOLD }, 64 { "ADD8rr_DB", "ADD8mr", NO_UNFOLD }, 65 { "ADD16rr_DB", "ADD16rm", NO_UNFOLD }, 66 { "ADD32rr_DB", "ADD32rm", NO_UNFOLD }, 67 { "ADD64rr_DB", "ADD64rm", NO_UNFOLD }, 68 { "ADD8rr_DB", "ADD8rm", NO_UNFOLD }, 69 { "PUSH16r", "PUSH16rmm", UNFOLD }, 70 { "PUSH32r", "PUSH32rmm", UNFOLD }, 71 { "PUSH64r", "PUSH64rmm", UNFOLD }, 72 { "TAILJMPr", "TAILJMPm", UNFOLD }, 73 { "TAILJMPr64", "TAILJMPm64", UNFOLD }, 74 { "TAILJMPr64_REX", "TAILJMPm64_REX", UNFOLD }, 75 }; 76 77 78 static bool isExplicitAlign(const CodeGenInstruction *Inst) { 79 return any_of(ExplicitAlign, [Inst](const char *InstStr) { 80 return Inst->TheDef->getName().contains(InstStr); 81 }); 82 } 83 84 static bool isExplicitUnalign(const CodeGenInstruction *Inst) { 85 return any_of(ExplicitUnalign, [Inst](const char *InstStr) { 86 return Inst->TheDef->getName().contains(InstStr); 87 }); 88 } 89 90 class X86FoldTablesEmitter { 91 RecordKeeper &Records; 92 CodeGenTarget Target; 93 94 // Represents an entry in the folding table 95 class X86FoldTableEntry { 96 const CodeGenInstruction *RegInst; 97 const CodeGenInstruction *MemInst; 98 99 public: 100 bool CannotUnfold = false; 101 bool IsLoad = false; 102 bool IsStore = false; 103 bool IsAligned = false; 104 unsigned int Alignment = 0; 105 106 X86FoldTableEntry(const CodeGenInstruction *RegInst, 107 const CodeGenInstruction *MemInst) 108 : RegInst(RegInst), MemInst(MemInst) {} 109 110 void print(formatted_raw_ostream &OS) const { 111 OS.indent(2); 112 OS << "{ X86::" << RegInst->TheDef->getName() << ","; 113 OS.PadToColumn(40); 114 OS << "X86::" << MemInst->TheDef->getName() << ","; 115 OS.PadToColumn(75); 116 117 if (IsLoad) 118 OS << "TB_FOLDED_LOAD | "; 119 if (IsStore) 120 OS << "TB_FOLDED_STORE | "; 121 if (CannotUnfold) 122 OS << "TB_NO_REVERSE | "; 123 if (IsAligned) 124 OS << "TB_ALIGN_" << Alignment << " | "; 125 126 OS << "0 },\n"; 127 } 128 129 bool operator<(const X86FoldTableEntry &RHS) const { 130 bool LHSpseudo = RegInst->TheDef->getValueAsBit("isPseudo"); 131 bool RHSpseudo = RHS.RegInst->TheDef->getValueAsBit("isPseudo"); 132 if (LHSpseudo != RHSpseudo) 133 return LHSpseudo; 134 135 return RegInst->TheDef->getName() < RHS.RegInst->TheDef->getName(); 136 } 137 }; 138 139 typedef std::vector<X86FoldTableEntry> FoldTable; 140 // std::vector for each folding table. 141 // Table2Addr - Holds instructions which their memory form performs load+store 142 // Table#i - Holds instructions which the their memory form perform a load OR 143 // a store, and their #i'th operand is folded. 144 FoldTable Table2Addr; 145 FoldTable Table0; 146 FoldTable Table1; 147 FoldTable Table2; 148 FoldTable Table3; 149 FoldTable Table4; 150 151 public: 152 X86FoldTablesEmitter(RecordKeeper &R) : Records(R), Target(R) {} 153 154 // run - Generate the 6 X86 memory fold tables. 155 void run(formatted_raw_ostream &OS); 156 157 private: 158 // Decides to which table to add the entry with the given instructions. 159 // S sets the strategy of adding the TB_NO_REVERSE flag. 160 void updateTables(const CodeGenInstruction *RegInstr, 161 const CodeGenInstruction *MemInstr, 162 const UnfoldStrategy S = NO_STRATEGY); 163 164 // Generates X86FoldTableEntry with the given instructions and fill it with 165 // the appropriate flags - then adds it to Table. 166 void addEntryWithFlags(FoldTable &Table, const CodeGenInstruction *RegInstr, 167 const CodeGenInstruction *MemInstr, 168 const UnfoldStrategy S, const unsigned int FoldedInd); 169 170 // Print the given table as a static const C++ array of type 171 // X86MemoryFoldTableEntry. 172 void printTable(const FoldTable &Table, StringRef TableName, 173 formatted_raw_ostream &OS) { 174 OS << "static const X86MemoryFoldTableEntry MemoryFold" << TableName 175 << "[] = {\n"; 176 177 for (const X86FoldTableEntry &E : Table) 178 E.print(OS); 179 180 OS << "};\n\n"; 181 } 182 }; 183 184 // Return true if one of the instruction's operands is a RST register class 185 static bool hasRSTRegClass(const CodeGenInstruction *Inst) { 186 return any_of(Inst->Operands, [](const CGIOperandList::OperandInfo &OpIn) { 187 return OpIn.Rec->getName() == "RST" || OpIn.Rec->getName() == "RSTi"; 188 }); 189 } 190 191 // Return true if one of the instruction's operands is a ptr_rc_tailcall 192 static bool hasPtrTailcallRegClass(const CodeGenInstruction *Inst) { 193 return any_of(Inst->Operands, [](const CGIOperandList::OperandInfo &OpIn) { 194 return OpIn.Rec->getName() == "ptr_rc_tailcall"; 195 }); 196 } 197 198 // Calculates the integer value representing the BitsInit object 199 static inline uint64_t getValueFromBitsInit(const BitsInit *B) { 200 assert(B->getNumBits() <= sizeof(uint64_t) * 8 && "BitInits' too long!"); 201 202 uint64_t Value = 0; 203 for (unsigned i = 0, e = B->getNumBits(); i != e; ++i) { 204 BitInit *Bit = cast<BitInit>(B->getBit(i)); 205 Value |= uint64_t(Bit->getValue()) << i; 206 } 207 return Value; 208 } 209 210 // Returns true if the two given BitsInits represent the same integer value 211 static inline bool equalBitsInits(const BitsInit *B1, const BitsInit *B2) { 212 if (B1->getNumBits() != B2->getNumBits()) 213 PrintFatalError("Comparing two BitsInits with different sizes!"); 214 215 for (unsigned i = 0, e = B1->getNumBits(); i != e; ++i) { 216 BitInit *Bit1 = cast<BitInit>(B1->getBit(i)); 217 BitInit *Bit2 = cast<BitInit>(B2->getBit(i)); 218 if (Bit1->getValue() != Bit2->getValue()) 219 return false; 220 } 221 return true; 222 } 223 224 // Return the size of the register operand 225 static inline unsigned int getRegOperandSize(const Record *RegRec) { 226 if (RegRec->isSubClassOf("RegisterOperand")) 227 RegRec = RegRec->getValueAsDef("RegClass"); 228 if (RegRec->isSubClassOf("RegisterClass")) 229 return RegRec->getValueAsListOfDefs("RegTypes")[0]->getValueAsInt("Size"); 230 231 llvm_unreachable("Register operand's size not known!"); 232 } 233 234 // Return the size of the memory operand 235 static inline unsigned getMemOperandSize(const Record *MemRec) { 236 if (MemRec->isSubClassOf("Operand")) { 237 StringRef Name = 238 MemRec->getValueAsDef("ParserMatchClass")->getValueAsString("Name"); 239 if (Name == "Mem8") 240 return 8; 241 if (Name == "Mem16") 242 return 16; 243 if (Name == "Mem32") 244 return 32; 245 if (Name == "Mem64") 246 return 64; 247 if (Name == "Mem80") 248 return 80; 249 if (Name == "Mem128") 250 return 128; 251 if (Name == "Mem256") 252 return 256; 253 if (Name == "Mem512") 254 return 512; 255 } 256 257 llvm_unreachable("Memory operand's size not known!"); 258 } 259 260 // Return true if the instruction defined as a register flavor. 261 static inline bool hasRegisterFormat(const Record *Inst) { 262 const BitsInit *FormBits = Inst->getValueAsBitsInit("FormBits"); 263 uint64_t FormBitsNum = getValueFromBitsInit(FormBits); 264 265 // Values from X86Local namespace defined in X86RecognizableInstr.cpp 266 return FormBitsNum >= X86Local::MRMDestReg && FormBitsNum <= X86Local::MRM7r; 267 } 268 269 // Return true if the instruction defined as a memory flavor. 270 static inline bool hasMemoryFormat(const Record *Inst) { 271 const BitsInit *FormBits = Inst->getValueAsBitsInit("FormBits"); 272 uint64_t FormBitsNum = getValueFromBitsInit(FormBits); 273 274 // Values from X86Local namespace defined in X86RecognizableInstr.cpp 275 return FormBitsNum >= X86Local::MRMDestMem && FormBitsNum <= X86Local::MRM7m; 276 } 277 278 static inline bool isNOREXRegClass(const Record *Op) { 279 return Op->getName().contains("_NOREX"); 280 } 281 282 static inline bool isRegisterOperand(const Record *Rec) { 283 return Rec->isSubClassOf("RegisterClass") || 284 Rec->isSubClassOf("RegisterOperand") || 285 Rec->isSubClassOf("PointerLikeRegClass"); 286 } 287 288 static inline bool isMemoryOperand(const Record *Rec) { 289 return Rec->isSubClassOf("Operand") && 290 Rec->getValueAsString("OperandType") == "OPERAND_MEMORY"; 291 } 292 293 static inline bool isImmediateOperand(const Record *Rec) { 294 return Rec->isSubClassOf("Operand") && 295 Rec->getValueAsString("OperandType") == "OPERAND_IMMEDIATE"; 296 } 297 298 // Get the alternative instruction pointed by "FoldGenRegForm" field. 299 static inline const CodeGenInstruction * 300 getAltRegInst(const CodeGenInstruction *I, const RecordKeeper &Records, 301 const CodeGenTarget &Target) { 302 303 StringRef AltRegInstStr = I->TheDef->getValueAsString("FoldGenRegForm"); 304 Record *AltRegInstRec = Records.getDef(AltRegInstStr); 305 assert(AltRegInstRec && 306 "Alternative register form instruction def not found"); 307 CodeGenInstruction &AltRegInst = Target.getInstruction(AltRegInstRec); 308 return &AltRegInst; 309 } 310 311 // Function object - Operator() returns true if the given VEX instruction 312 // matches the EVEX instruction of this object. 313 class IsMatch { 314 const CodeGenInstruction *MemInst; 315 316 public: 317 IsMatch(const CodeGenInstruction *Inst, const RecordKeeper &Records) 318 : MemInst(Inst) {} 319 320 bool operator()(const CodeGenInstruction *RegInst) { 321 Record *MemRec = MemInst->TheDef; 322 Record *RegRec = RegInst->TheDef; 323 324 // Return false if one (at least) of the encoding fields of both 325 // instructions do not match. 326 if (RegRec->getValueAsDef("OpEnc") != MemRec->getValueAsDef("OpEnc") || 327 !equalBitsInits(RegRec->getValueAsBitsInit("Opcode"), 328 MemRec->getValueAsBitsInit("Opcode")) || 329 // VEX/EVEX fields 330 RegRec->getValueAsDef("OpPrefix") != 331 MemRec->getValueAsDef("OpPrefix") || 332 RegRec->getValueAsDef("OpMap") != MemRec->getValueAsDef("OpMap") || 333 RegRec->getValueAsDef("OpSize") != MemRec->getValueAsDef("OpSize") || 334 RegRec->getValueAsDef("AdSize") != MemRec->getValueAsDef("AdSize") || 335 RegRec->getValueAsBit("hasVEX_4V") != 336 MemRec->getValueAsBit("hasVEX_4V") || 337 RegRec->getValueAsBit("hasEVEX_K") != 338 MemRec->getValueAsBit("hasEVEX_K") || 339 RegRec->getValueAsBit("hasEVEX_Z") != 340 MemRec->getValueAsBit("hasEVEX_Z") || 341 // EVEX_B means different things for memory and register forms. 342 RegRec->getValueAsBit("hasEVEX_B") != 0 || 343 MemRec->getValueAsBit("hasEVEX_B") != 0 || 344 RegRec->getValueAsBit("hasEVEX_RC") != 345 MemRec->getValueAsBit("hasEVEX_RC") || 346 RegRec->getValueAsBit("hasREX_WPrefix") != 347 MemRec->getValueAsBit("hasREX_WPrefix") || 348 RegRec->getValueAsBit("hasLockPrefix") != 349 MemRec->getValueAsBit("hasLockPrefix") || 350 RegRec->getValueAsBit("hasNoTrackPrefix") != 351 MemRec->getValueAsBit("hasNoTrackPrefix") || 352 RegRec->getValueAsBit("hasVEX_L") != 353 MemRec->getValueAsBit("hasVEX_L") || 354 RegRec->getValueAsBit("hasEVEX_L2") != 355 MemRec->getValueAsBit("hasEVEX_L2") || 356 RegRec->getValueAsBit("ignoresVEX_L") != 357 MemRec->getValueAsBit("ignoresVEX_L") || 358 RegRec->getValueAsBit("HasVEX_W") != 359 MemRec->getValueAsBit("HasVEX_W") || 360 RegRec->getValueAsBit("IgnoresVEX_W") != 361 MemRec->getValueAsBit("IgnoresVEX_W") || 362 RegRec->getValueAsBit("EVEX_W1_VEX_W0") != 363 MemRec->getValueAsBit("EVEX_W1_VEX_W0") || 364 // Instruction's format - The register form's "Form" field should be 365 // the opposite of the memory form's "Form" field. 366 !areOppositeForms(RegRec->getValueAsBitsInit("FormBits"), 367 MemRec->getValueAsBitsInit("FormBits")) || 368 RegRec->getValueAsBit("isAsmParserOnly") != 369 MemRec->getValueAsBit("isAsmParserOnly")) 370 return false; 371 372 // Make sure the sizes of the operands of both instructions suit each other. 373 // This is needed for instructions with intrinsic version (_Int). 374 // Where the only difference is the size of the operands. 375 // For example: VUCOMISDZrm and Int_VUCOMISDrm 376 // Also for instructions that their EVEX version was upgraded to work with 377 // k-registers. For example VPCMPEQBrm (xmm output register) and 378 // VPCMPEQBZ128rm (k register output register). 379 bool ArgFolded = false; 380 unsigned MemOutSize = MemRec->getValueAsDag("OutOperandList")->getNumArgs(); 381 unsigned RegOutSize = RegRec->getValueAsDag("OutOperandList")->getNumArgs(); 382 unsigned MemInSize = MemRec->getValueAsDag("InOperandList")->getNumArgs(); 383 unsigned RegInSize = RegRec->getValueAsDag("InOperandList")->getNumArgs(); 384 385 // Instructions with one output in their memory form use the memory folded 386 // operand as source and destination (Read-Modify-Write). 387 unsigned RegStartIdx = 388 (MemOutSize + 1 == RegOutSize) && (MemInSize == RegInSize) ? 1 : 0; 389 390 for (unsigned i = 0, e = MemInst->Operands.size(); i < e; i++) { 391 Record *MemOpRec = MemInst->Operands[i].Rec; 392 Record *RegOpRec = RegInst->Operands[i + RegStartIdx].Rec; 393 394 if (MemOpRec == RegOpRec) 395 continue; 396 397 if (isRegisterOperand(MemOpRec) && isRegisterOperand(RegOpRec)) { 398 if (getRegOperandSize(MemOpRec) != getRegOperandSize(RegOpRec) || 399 isNOREXRegClass(MemOpRec) != isNOREXRegClass(RegOpRec)) 400 return false; 401 } else if (isMemoryOperand(MemOpRec) && isMemoryOperand(RegOpRec)) { 402 if (getMemOperandSize(MemOpRec) != getMemOperandSize(RegOpRec)) 403 return false; 404 } else if (isImmediateOperand(MemOpRec) && isImmediateOperand(RegOpRec)) { 405 if (MemOpRec->getValueAsDef("Type") != RegOpRec->getValueAsDef("Type")) 406 return false; 407 } else { 408 // Only one operand can be folded. 409 if (ArgFolded) 410 return false; 411 412 assert(isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec)); 413 ArgFolded = true; 414 } 415 } 416 417 return true; 418 } 419 420 private: 421 // Return true of the 2 given forms are the opposite of each other. 422 bool areOppositeForms(const BitsInit *RegFormBits, 423 const BitsInit *MemFormBits) { 424 uint64_t MemFormNum = getValueFromBitsInit(MemFormBits); 425 uint64_t RegFormNum = getValueFromBitsInit(RegFormBits); 426 427 if ((MemFormNum == X86Local::MRM0m && RegFormNum == X86Local::MRM0r) || 428 (MemFormNum == X86Local::MRM1m && RegFormNum == X86Local::MRM1r) || 429 (MemFormNum == X86Local::MRM2m && RegFormNum == X86Local::MRM2r) || 430 (MemFormNum == X86Local::MRM3m && RegFormNum == X86Local::MRM3r) || 431 (MemFormNum == X86Local::MRM4m && RegFormNum == X86Local::MRM4r) || 432 (MemFormNum == X86Local::MRM5m && RegFormNum == X86Local::MRM5r) || 433 (MemFormNum == X86Local::MRM6m && RegFormNum == X86Local::MRM6r) || 434 (MemFormNum == X86Local::MRM7m && RegFormNum == X86Local::MRM7r) || 435 (MemFormNum == X86Local::MRMXm && RegFormNum == X86Local::MRMXr) || 436 (MemFormNum == X86Local::MRMXmCC && RegFormNum == X86Local::MRMXrCC) || 437 (MemFormNum == X86Local::MRMDestMem && 438 RegFormNum == X86Local::MRMDestReg) || 439 (MemFormNum == X86Local::MRMSrcMem && 440 RegFormNum == X86Local::MRMSrcReg) || 441 (MemFormNum == X86Local::MRMSrcMem4VOp3 && 442 RegFormNum == X86Local::MRMSrcReg4VOp3) || 443 (MemFormNum == X86Local::MRMSrcMemOp4 && 444 RegFormNum == X86Local::MRMSrcRegOp4) || 445 (MemFormNum == X86Local::MRMSrcMemCC && 446 RegFormNum == X86Local::MRMSrcRegCC)) 447 return true; 448 449 return false; 450 } 451 }; 452 453 } // end anonymous namespace 454 455 void X86FoldTablesEmitter::addEntryWithFlags(FoldTable &Table, 456 const CodeGenInstruction *RegInstr, 457 const CodeGenInstruction *MemInstr, 458 const UnfoldStrategy S, 459 const unsigned int FoldedInd) { 460 461 X86FoldTableEntry Result = X86FoldTableEntry(RegInstr, MemInstr); 462 Record *RegRec = RegInstr->TheDef; 463 Record *MemRec = MemInstr->TheDef; 464 465 // Only table0 entries should explicitly specify a load or store flag. 466 if (&Table == &Table0) { 467 unsigned MemInOpsNum = MemRec->getValueAsDag("InOperandList")->getNumArgs(); 468 unsigned RegInOpsNum = RegRec->getValueAsDag("InOperandList")->getNumArgs(); 469 // If the instruction writes to the folded operand, it will appear as an 470 // output in the register form instruction and as an input in the memory 471 // form instruction. 472 // If the instruction reads from the folded operand, it well appear as in 473 // input in both forms. 474 if (MemInOpsNum == RegInOpsNum) 475 Result.IsLoad = true; 476 else 477 Result.IsStore = true; 478 } 479 480 Record *RegOpRec = RegInstr->Operands[FoldedInd].Rec; 481 Record *MemOpRec = MemInstr->Operands[FoldedInd].Rec; 482 483 // Unfolding code generates a load/store instruction according to the size of 484 // the register in the register form instruction. 485 // If the register's size is greater than the memory's operand size, do not 486 // allow unfolding. 487 if (S == UNFOLD) 488 Result.CannotUnfold = false; 489 else if (S == NO_UNFOLD) 490 Result.CannotUnfold = true; 491 else if (getRegOperandSize(RegOpRec) > getMemOperandSize(MemOpRec)) 492 Result.CannotUnfold = true; // S == NO_STRATEGY 493 494 uint64_t Enc = getValueFromBitsInit(RegRec->getValueAsBitsInit("OpEncBits")); 495 if (isExplicitAlign(RegInstr)) { 496 // The instruction require explicitly aligned memory. 497 BitsInit *VectSize = RegRec->getValueAsBitsInit("VectSize"); 498 uint64_t Value = getValueFromBitsInit(VectSize); 499 Result.IsAligned = true; 500 Result.Alignment = Value; 501 } else if (Enc != X86Local::XOP && Enc != X86Local::VEX && 502 Enc != X86Local::EVEX) { 503 // Instructions with VEX encoding do not require alignment. 504 if (!isExplicitUnalign(RegInstr) && getMemOperandSize(MemOpRec) > 64) { 505 // SSE packed vector instructions require a 16 byte alignment. 506 Result.IsAligned = true; 507 Result.Alignment = 16; 508 } 509 } 510 511 Table.push_back(Result); 512 } 513 514 void X86FoldTablesEmitter::updateTables(const CodeGenInstruction *RegInstr, 515 const CodeGenInstruction *MemInstr, 516 const UnfoldStrategy S) { 517 518 Record *RegRec = RegInstr->TheDef; 519 Record *MemRec = MemInstr->TheDef; 520 unsigned MemOutSize = MemRec->getValueAsDag("OutOperandList")->getNumArgs(); 521 unsigned RegOutSize = RegRec->getValueAsDag("OutOperandList")->getNumArgs(); 522 unsigned MemInSize = MemRec->getValueAsDag("InOperandList")->getNumArgs(); 523 unsigned RegInSize = RegRec->getValueAsDag("InOperandList")->getNumArgs(); 524 525 // Instructions which Read-Modify-Write should be added to Table2Addr. 526 if (MemOutSize != RegOutSize && MemInSize == RegInSize) { 527 addEntryWithFlags(Table2Addr, RegInstr, MemInstr, S, 0); 528 return; 529 } 530 531 if (MemInSize == RegInSize && MemOutSize == RegOutSize) { 532 // Load-Folding cases. 533 // If the i'th register form operand is a register and the i'th memory form 534 // operand is a memory operand, add instructions to Table#i. 535 for (unsigned i = RegOutSize, e = RegInstr->Operands.size(); i < e; i++) { 536 Record *RegOpRec = RegInstr->Operands[i].Rec; 537 Record *MemOpRec = MemInstr->Operands[i].Rec; 538 if (isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec)) { 539 switch (i) { 540 case 0: 541 addEntryWithFlags(Table0, RegInstr, MemInstr, S, 0); 542 return; 543 case 1: 544 addEntryWithFlags(Table1, RegInstr, MemInstr, S, 1); 545 return; 546 case 2: 547 addEntryWithFlags(Table2, RegInstr, MemInstr, S, 2); 548 return; 549 case 3: 550 addEntryWithFlags(Table3, RegInstr, MemInstr, S, 3); 551 return; 552 case 4: 553 addEntryWithFlags(Table4, RegInstr, MemInstr, S, 4); 554 return; 555 } 556 } 557 } 558 } else if (MemInSize == RegInSize + 1 && MemOutSize + 1 == RegOutSize) { 559 // Store-Folding cases. 560 // If the memory form instruction performs a store, the *output* 561 // register of the register form instructions disappear and instead a 562 // memory *input* operand appears in the memory form instruction. 563 // For example: 564 // MOVAPSrr => (outs VR128:$dst), (ins VR128:$src) 565 // MOVAPSmr => (outs), (ins f128mem:$dst, VR128:$src) 566 Record *RegOpRec = RegInstr->Operands[RegOutSize - 1].Rec; 567 Record *MemOpRec = MemInstr->Operands[RegOutSize - 1].Rec; 568 if (isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec) && 569 getRegOperandSize(RegOpRec) == getMemOperandSize(MemOpRec)) 570 addEntryWithFlags(Table0, RegInstr, MemInstr, S, 0); 571 } 572 } 573 574 void X86FoldTablesEmitter::run(formatted_raw_ostream &OS) { 575 emitSourceFileHeader("X86 fold tables", OS); 576 577 // Holds all memory instructions 578 std::vector<const CodeGenInstruction *> MemInsts; 579 // Holds all register instructions - divided according to opcode. 580 std::map<uint8_t, std::vector<const CodeGenInstruction *>> RegInsts; 581 582 ArrayRef<const CodeGenInstruction *> NumberedInstructions = 583 Target.getInstructionsByEnumValue(); 584 585 for (const CodeGenInstruction *Inst : NumberedInstructions) { 586 if (!Inst->TheDef->getNameInit() || !Inst->TheDef->isSubClassOf("X86Inst")) 587 continue; 588 589 const Record *Rec = Inst->TheDef; 590 591 // - Do not proceed if the instruction is marked as notMemoryFoldable. 592 // - Instructions including RST register class operands are not relevant 593 // for memory folding (for further details check the explanation in 594 // lib/Target/X86/X86InstrFPStack.td file). 595 // - Some instructions (listed in the manual map above) use the register 596 // class ptr_rc_tailcall, which can be of a size 32 or 64, to ensure 597 // safe mapping of these instruction we manually map them and exclude 598 // them from the automation. 599 if (Rec->getValueAsBit("isMemoryFoldable") == false || 600 hasRSTRegClass(Inst) || hasPtrTailcallRegClass(Inst)) 601 continue; 602 603 // Add all the memory form instructions to MemInsts, and all the register 604 // form instructions to RegInsts[Opc], where Opc in the opcode of each 605 // instructions. this helps reducing the runtime of the backend. 606 if (hasMemoryFormat(Rec)) 607 MemInsts.push_back(Inst); 608 else if (hasRegisterFormat(Rec)) { 609 uint8_t Opc = getValueFromBitsInit(Rec->getValueAsBitsInit("Opcode")); 610 RegInsts[Opc].push_back(Inst); 611 } 612 } 613 614 // For each memory form instruction, try to find its register form 615 // instruction. 616 for (const CodeGenInstruction *MemInst : MemInsts) { 617 uint8_t Opc = 618 getValueFromBitsInit(MemInst->TheDef->getValueAsBitsInit("Opcode")); 619 620 auto RegInstsIt = RegInsts.find(Opc); 621 if (RegInstsIt == RegInsts.end()) 622 continue; 623 624 // Two forms (memory & register) of the same instruction must have the same 625 // opcode. try matching only with register form instructions with the same 626 // opcode. 627 std::vector<const CodeGenInstruction *> &OpcRegInsts = RegInstsIt->second; 628 629 auto Match = find_if(OpcRegInsts, IsMatch(MemInst, Records)); 630 if (Match != OpcRegInsts.end()) { 631 const CodeGenInstruction *RegInst = *Match; 632 // If the matched instruction has it's "FoldGenRegForm" set, map the 633 // memory form instruction to the register form instruction pointed by 634 // this field 635 if (RegInst->TheDef->isValueUnset("FoldGenRegForm")) { 636 updateTables(RegInst, MemInst); 637 } else { 638 const CodeGenInstruction *AltRegInst = 639 getAltRegInst(RegInst, Records, Target); 640 updateTables(AltRegInst, MemInst); 641 } 642 OpcRegInsts.erase(Match); 643 } 644 } 645 646 // Add the manually mapped instructions listed above. 647 for (const ManualMapEntry &Entry : ManualMapSet) { 648 Record *RegInstIter = Records.getDef(Entry.RegInstStr); 649 Record *MemInstIter = Records.getDef(Entry.MemInstStr); 650 651 updateTables(&(Target.getInstruction(RegInstIter)), 652 &(Target.getInstruction(MemInstIter)), Entry.Strategy); 653 } 654 655 // Sort the tables before printing. 656 llvm::sort(Table2Addr); 657 llvm::sort(Table0); 658 llvm::sort(Table1); 659 llvm::sort(Table2); 660 llvm::sort(Table3); 661 llvm::sort(Table4); 662 663 // Print all tables. 664 printTable(Table2Addr, "Table2Addr", OS); 665 printTable(Table0, "Table0", OS); 666 printTable(Table1, "Table1", OS); 667 printTable(Table2, "Table2", OS); 668 printTable(Table3, "Table3", OS); 669 printTable(Table4, "Table4", OS); 670 } 671 672 namespace llvm { 673 674 void EmitX86FoldTables(RecordKeeper &RK, raw_ostream &o) { 675 formatted_raw_ostream OS(o); 676 X86FoldTablesEmitter(RK).run(OS); 677 } 678 } // namespace llvm 679