Lines Matching +full:adc +full:- +full:tm

1 //===------- X86ExpandPseudo.cpp - Expand pseudo instructions -------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 // instructions to allow proper scheduling, if-conversion, other late
13 //===----------------------------------------------------------------------===//
30 #define DEBUG_TYPE "x86-pseudo"
94 MachineFunction *MF = MBB->getParent(); in INITIALIZE_PASS()
95 const BasicBlock *BB = MBB->getBasicBlock(); in INITIALIZE_PASS()
100 const DebugLoc &DL = JTInst->getDebugLoc(); in INITIALIZE_PASS()
101 MachineOperand Selector = JTInst->getOperand(0); in INITIALIZE_PASS()
102 const GlobalValue *CombinedGlobal = JTInst->getOperand(1).getGlobal(); in INITIALIZE_PASS()
106 MBB->addLiveIn(Selector.getReg()); in INITIALIZE_PASS()
107 BuildMI(*MBB, MBBI, DL, TII->get(X86::LEA64r), X86::R11) in INITIALIZE_PASS()
112 JTInst->getOperand(2 + 2 * Target).getImm()) in INITIALIZE_PASS()
114 BuildMI(*MBB, MBBI, DL, TII->get(X86::CMP64rr)) in INITIALIZE_PASS()
120 auto *NewMBB = MF->CreateMachineBasicBlock(BB); in INITIALIZE_PASS()
121 MBB->addSuccessor(NewMBB); in INITIALIZE_PASS()
122 if (!MBB->isLiveIn(X86::EFLAGS)) in INITIALIZE_PASS()
123 MBB->addLiveIn(X86::EFLAGS); in INITIALIZE_PASS()
128 BuildMI(*MBB, MBBI, DL, TII->get(X86::JCC_1)).addMBB(ThenMBB).addImm(CC); in INITIALIZE_PASS()
131 MF->insert(InsPt, ElseMBB); in INITIALIZE_PASS()
133 MBBI = MBB->end(); in INITIALIZE_PASS()
143 BuildMI(*MBB, MBBI, DL, TII->get(X86::TAILJMPd64)) in INITIALIZE_PASS()
144 .add(JTInst->getOperand(3 + 2 * Target)); in INITIALIZE_PASS()
165 EmitBranchFunnel(FirstTarget + 2, NumTargets - 2); in INITIALIZE_PASS()
174 NumTargets - (NumTargets / 2) - 1); in INITIALIZE_PASS()
176 MF->insert(InsPt, ThenMBB); in INITIALIZE_PASS()
178 MBBI = MBB->end(); in INITIALIZE_PASS()
182 EmitBranchFunnel(0, (JTInst->getNumOperands() - 2) / 2); in INITIALIZE_PASS()
184 MF->insert(InsPt, P.first); in INITIALIZE_PASS()
185 BuildMI(P.first, DL, TII->get(X86::TAILJMPd64)) in INITIALIZE_PASS()
186 .add(JTInst->getOperand(3 + 2 * P.second)); in INITIALIZE_PASS()
188 JTMBB->erase(JTInst); in INITIALIZE_PASS()
200 unsigned Opc = -1; in expandCALL_RVMARKER()
210 OriginalCall = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc)).getInstr(); in expandCALL_RVMARKER()
216 TRI->regsOverlap(Op.getReg(), X86::RAX)) { in expandCALL_RVMARKER()
221 OriginalCall->addOperand(Op); in expandCALL_RVMARKER()
224 // Emit marker "movq %rax, %rdi". %rdi is not callee-saved, so it cannot be in expandCALL_RVMARKER()
229 auto TargetReg = STI->getTargetTriple().isOSWindows() ? X86::RCX : X86::RDI; in expandCALL_RVMARKER()
230 auto *Marker = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(X86::MOV64rr)) in expandCALL_RVMARKER()
235 MBB.getParent()->moveCallSiteInfo(&MI, Marker); in expandCALL_RVMARKER()
239 TRI->getCallPreservedMask(*MBB.getParent(), CallingConv::C); in expandCALL_RVMARKER()
241 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(X86::CALL64pcrel32)) in expandCALL_RVMARKER()
251 auto &TM = MBB.getParent()->getTarget(); in expandCALL_RVMARKER() local
254 if (TM.getTargetTriple().isOSDarwin()) in expandCALL_RVMARKER()
255 finalizeBundle(MBB, OriginalCall->getIterator(), in expandCALL_RVMARKER()
256 std::next(RtCall->getIterator())); in expandCALL_RVMARKER()
266 const DebugLoc &DL = MBBI->getDebugLoc(); in expandMI()
267 #define GET_EGPR_IF_ENABLED(OPC) (STI->hasEGPR() ? OPC##_EVEX : OPC) in expandMI()
280 MachineOperand &JumpTarget = MBBI->getOperand(0); in expandMI()
281 MachineOperand &StackAdjust = MBBI->getOperand(isMem ? X86::AddrNumOperands in expandMI()
287 int MaxTCDelta = X86FI->getTCReturnAddrDelta(); in expandMI()
292 Offset = StackAdj - MaxTCDelta; in expandMI()
301 Offset += X86FL->mergeSPUpdates(MBB, MBBI, true); in expandMI()
302 X86FL->emitSPUpdate(MBB, MBBI, DL, Offset, /*InEpilogue=*/true); in expandMI()
306 bool IsWin64 = STI->isTargetWin64(); in expandMI()
318 assert(!MBB.getParent()->hasWinCFI() && in expandMI()
329 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(Op)); in expandMI()
339 MIB.addImm(MBBI->getOperand(2).getImm()); in expandMI()
346 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(Op)); in expandMI()
348 MIB.add(MBBI->getOperand(i)); in expandMI()
352 TII->get(IsWin64 ? X86::TAILJMPr64_REX : X86::TAILJMPr64)) in expandMI()
356 BuildMI(MBB, MBBI, DL, TII->get(X86::TAILJMPr)) in expandMI()
361 NewMI.copyImplicitOps(*MBBI->getParent()->getParent(), *MBBI); in expandMI()
365 if (MBBI->isCandidateForCallSiteEntry()) in expandMI()
366 MBB.getParent()->moveCallSiteInfo(&*MBBI, &NewMI); in expandMI()
375 MachineOperand &DestAddr = MBBI->getOperand(0); in expandMI()
378 STI->isTarget64BitLP64() || STI->isTargetNaCl64(); in expandMI()
379 Register StackPtr = TRI->getStackRegister(); in expandMI()
381 TII->get(Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr), StackPtr) in expandMI()
388 int64_t StackAdj = MBBI->getOperand(0).getImm(); in expandMI()
389 X86FL->emitSPUpdate(MBB, MBBI, DL, StackAdj, true); in expandMI()
391 unsigned RetOp = STI->is64Bit() ? X86::IRET64 : X86::IRET32; in expandMI()
393 if (STI->is64Bit() && STI->hasUINTR() && in expandMI()
394 MBB.getParent()->getTarget().getCodeModel() != CodeModel::Kernel) in expandMI()
396 BuildMI(MBB, MBBI, DL, TII->get(RetOp)); in expandMI()
402 int64_t StackAdj = MBBI->getOperand(0).getImm(); in expandMI()
406 TII->get(STI->is64Bit() ? X86::RET64 : X86::RET32)); in expandMI()
409 TII->get(STI->is64Bit() ? X86::RETI64 : X86::RETI32)) in expandMI()
412 assert(!STI->is64Bit() && in expandMI()
414 // A ret can only handle immediates as big as 2**16-1. If we need to pop in expandMI()
416 BuildMI(MBB, MBBI, DL, TII->get(X86::POP32r)).addReg(X86::ECX, RegState::Define); in expandMI()
417 X86FL->emitSPUpdate(MBB, MBBI, DL, StackAdj, /*InEpilogue=*/true); in expandMI()
418 BuildMI(MBB, MBBI, DL, TII->get(X86::PUSH32r)).addReg(X86::ECX); in expandMI()
419 MIB = BuildMI(MBB, MBBI, DL, TII->get(X86::RET32)); in expandMI()
421 for (unsigned I = 1, E = MBBI->getNumOperands(); I != E; ++I) in expandMI()
422 MIB.add(MBBI->getOperand(I)); in expandMI()
433 const MachineOperand &InArg = MBBI->getOperand(6); in expandMI()
434 Register SaveRbx = MBBI->getOperand(7).getReg(); in expandMI()
440 TII->copyPhysReg(MBB, MBBI, DL, X86::RBX, InArg.getReg(), false); in expandMI()
442 MachineInstr *NewInstr = BuildMI(MBB, MBBI, DL, TII->get(X86::LCMPXCHG16B)); in expandMI()
445 NewInstr->addOperand(MBBI->getOperand(Idx)); in expandMI()
447 TII->copyPhysReg(MBB, MBBI, DL, X86::RBX, SaveRbx, in expandMI()
451 MBBI->eraseFromParent(); in expandMI()
463 int64_t Disp = MBBI->getOperand(1 + X86::AddrDisp).getImm(); in expandMI()
464 assert(Disp >= 0 && Disp <= INT32_MAX - 2 && "Unexpected displacement"); in expandMI()
465 Register Reg = MBBI->getOperand(0).getReg(); in expandMI()
466 bool DstIsDead = MBBI->getOperand(0).isDead(); in expandMI()
467 Register Reg0 = TRI->getSubReg(Reg, X86::sub_mask_0); in expandMI()
468 Register Reg1 = TRI->getSubReg(Reg, X86::sub_mask_1); in expandMI()
471 BuildMI(MBB, MBBI, DL, TII->get(GET_EGPR_IF_ENABLED(X86::KMOVWkm))) in expandMI()
474 BuildMI(MBB, MBBI, DL, TII->get(GET_EGPR_IF_ENABLED(X86::KMOVWkm))) in expandMI()
478 MIBLo.add(MBBI->getOperand(1 + i)); in expandMI()
482 MIBHi.add(MBBI->getOperand(1 + i)); in expandMI()
486 MachineMemOperand *OldMMO = MBBI->memoperands().front(); in expandMI()
488 MachineMemOperand *MMOLo = MF->getMachineMemOperand(OldMMO, 0, 2); in expandMI()
489 MachineMemOperand *MMOHi = MF->getMachineMemOperand(OldMMO, 2, 2); in expandMI()
499 int64_t Disp = MBBI->getOperand(X86::AddrDisp).getImm(); in expandMI()
500 assert(Disp >= 0 && Disp <= INT32_MAX - 2 && "Unexpected displacement"); in expandMI()
501 Register Reg = MBBI->getOperand(X86::AddrNumOperands).getReg(); in expandMI()
502 bool SrcIsKill = MBBI->getOperand(X86::AddrNumOperands).isKill(); in expandMI()
503 Register Reg0 = TRI->getSubReg(Reg, X86::sub_mask_0); in expandMI()
504 Register Reg1 = TRI->getSubReg(Reg, X86::sub_mask_1); in expandMI()
507 BuildMI(MBB, MBBI, DL, TII->get(GET_EGPR_IF_ENABLED(X86::KMOVWmk))); in expandMI()
509 BuildMI(MBB, MBBI, DL, TII->get(GET_EGPR_IF_ENABLED(X86::KMOVWmk))); in expandMI()
512 MIBLo.add(MBBI->getOperand(i)); in expandMI()
516 MIBHi.add(MBBI->getOperand(i)); in expandMI()
522 MachineMemOperand *OldMMO = MBBI->memoperands().front(); in expandMI()
524 MachineMemOperand *MMOLo = MF->getMachineMemOperand(OldMMO, 0, 2); in expandMI()
525 MachineMemOperand *MMOHi = MF->getMachineMemOperand(OldMMO, 2, 2); in expandMI()
541 const MachineOperand &InArg = MBBI->getOperand(1); in expandMI()
544 TII->copyPhysReg(MBB, MBBI, DL, X86::EBX, InArg.getReg(), InArg.isKill()); in expandMI()
546 BuildMI(MBB, MBBI, DL, TII->get(X86::MWAITXrrr)); in expandMI()
548 Register SaveRbx = MBBI->getOperand(2).getReg(); in expandMI()
549 TII->copyPhysReg(MBB, MBBI, DL, X86::RBX, SaveRbx, /*SrcIsKill*/ true); in expandMI()
551 MBBI->eraseFromParent(); in expandMI()
558 MI.setDesc(TII->get(GET_EGPR_IF_ENABLED(X86::LDTILECFG))); in expandMI()
563 for (unsigned i = 2; i > 0; --i) in expandMI()
568 MI.setDesc(TII->get(Opc)); in expandMI()
580 for (unsigned i = 3; i > 0; --i) in expandMI()
594 MI.setDesc(TII->get(Opc)); in expandMI()
599 for (int i = 1; i >= 0; --i) in expandMI()
601 MI.setDesc(TII->get(GET_EGPR_IF_ENABLED(X86::TILESTORED))); in expandMI()
606 for (int i = 2; i > 0; --i) // Remove row, col in expandMI()
608 MI.setDesc(TII->get(X86::TILEZERO)); in expandMI()
630 // It's possible for an EVEX-encoded legacy instruction to reach the 15-byte in expandMI()
647 MI.getOperand(MI.getNumExplicitOperands() - 1); in expandMI()
686 MI_TO_RI(ADC); in expandMI()
692 BuildMI(MBB, std::next(MBBI), DL, TII->get(Opc), DestReg) in expandMI()
696 for (unsigned I = MI.getNumImplicitOperands() + 1; I != 0; --I) in expandMI()
697 MI.removeOperand(MI.getNumOperands() - 1); in expandMI()
698 MI.setDesc(TII->get(LoadOpc)); in expandMI()
722 assert(VAStartPseudoInstr->getOpcode() == X86::VASTART_SAVE_XMM_REGS); in expandVastartSaveXmmRegs()
724 MachineFunction *Func = EntryBlk->getParent(); in expandVastartSaveXmmRegs()
725 const TargetInstrInfo *TII = STI->getInstrInfo(); in expandVastartSaveXmmRegs()
726 const DebugLoc &DL = VAStartPseudoInstr->getDebugLoc(); in expandVastartSaveXmmRegs()
727 Register CountReg = VAStartPseudoInstr->getOperand(0).getReg(); in expandVastartSaveXmmRegs()
730 LivePhysRegs LiveRegs(*STI->getRegisterInfo()); in expandVastartSaveXmmRegs()
734 for (MachineInstr &MI : EntryBlk->instrs()) { in expandVastartSaveXmmRegs()
735 if (MI.getOpcode() == VAStartPseudoInstr->getOpcode()) in expandVastartSaveXmmRegs()
744 const BasicBlock *LLVMBlk = EntryBlk->getBasicBlock(); in expandVastartSaveXmmRegs()
745 MachineFunction::iterator EntryBlkIter = ++EntryBlk->getIterator(); in expandVastartSaveXmmRegs()
746 MachineBasicBlock *GuardedRegsBlk = Func->CreateMachineBasicBlock(LLVMBlk); in expandVastartSaveXmmRegs()
747 MachineBasicBlock *TailBlk = Func->CreateMachineBasicBlock(LLVMBlk); in expandVastartSaveXmmRegs()
748 Func->insert(EntryBlkIter, GuardedRegsBlk); in expandVastartSaveXmmRegs()
749 Func->insert(EntryBlkIter, TailBlk); in expandVastartSaveXmmRegs()
752 TailBlk->splice(TailBlk->begin(), EntryBlk, in expandVastartSaveXmmRegs()
754 EntryBlk->end()); in expandVastartSaveXmmRegs()
755 TailBlk->transferSuccessorsAndUpdatePHIs(EntryBlk); in expandVastartSaveXmmRegs()
757 uint64_t FrameOffset = VAStartPseudoInstr->getOperand(4).getImm(); in expandVastartSaveXmmRegs()
758 uint64_t VarArgsRegsOffset = VAStartPseudoInstr->getOperand(6).getImm(); in expandVastartSaveXmmRegs()
761 unsigned MOVOpc = STI->hasAVX() ? X86::VMOVAPSmr : X86::MOVAPSmr; in expandVastartSaveXmmRegs()
765 OpndIdx < VAStartPseudoInstr->getNumOperands() - 1; in expandVastartSaveXmmRegs()
767 auto NewMI = BuildMI(GuardedRegsBlk, DL, TII->get(MOVOpc)); in expandVastartSaveXmmRegs()
772 NewMI.add(VAStartPseudoInstr->getOperand(i + 1)); in expandVastartSaveXmmRegs()
774 NewMI.addReg(VAStartPseudoInstr->getOperand(OpndIdx).getReg()); in expandVastartSaveXmmRegs()
775 assert(VAStartPseudoInstr->getOperand(OpndIdx).getReg().isPhysical()); in expandVastartSaveXmmRegs()
779 EntryBlk->addSuccessor(GuardedRegsBlk); in expandVastartSaveXmmRegs()
781 GuardedRegsBlk->addSuccessor(TailBlk); in expandVastartSaveXmmRegs()
783 if (!STI->isCallingConvWin64(Func->getFunction().getCallingConv())) { in expandVastartSaveXmmRegs()
785 BuildMI(EntryBlk, DL, TII->get(X86::TEST8rr)) in expandVastartSaveXmmRegs()
788 BuildMI(EntryBlk, DL, TII->get(X86::JCC_1)) in expandVastartSaveXmmRegs()
791 EntryBlk->addSuccessor(TailBlk); in expandVastartSaveXmmRegs()
799 VAStartPseudoInstr->eraseFromParent(); in expandVastartSaveXmmRegs()
834 TII = STI->getInstrInfo(); in runOnMachineFunction()
835 TRI = STI->getRegisterInfo(); in runOnMachineFunction()
837 X86FL = STI->getFrameLowering(); in runOnMachineFunction()