Lines Matching +full:guest +full:- +full:side
1 //===-- AArch64Arm64ECCallLowering.cpp - Lower Arm64EC calls ----*- C++ -*-===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
16 //===----------------------------------------------------------------------===//
44 static cl::opt<bool> LowerDirectToIndirect("arm64ec-lower-direct-to-indirect",
46 static cl::opt<bool> GenerateThunks("arm64ec-generate-thunks", cl::Hidden,
132 // for entry/guest exit thunks, we just call the Arm64 function directly. in getThunkType()
156 if (FT->isVarArg()) { in getThunkArgTypes()
158 // with the following type on the ARM side: in getThunkArgTypes()
164 // x0-x3 is the arguments be stored in registers. in getThunkArgTypes()
168 // On the x64 side, it's the same except that x5 isn't set. in getThunkArgTypes()
173 // If the X64 side is sret, but the ARM side isn't, we pass an extra value in getThunkArgTypes()
174 // to/from the X64 side, and let SelectionDAG transform it into a memory in getThunkArgTypes()
178 // x0-x3 in getThunkArgTypes()
192 // FIXME: x5 isn't actually used by the x64 side; revisit once we in getThunkArgTypes()
204 if (I == FT->getNumParams()) { in getThunkArgTypes()
209 for (unsigned E = FT->getNumParams(); I != E; ++I) { in getThunkArgTypes()
220 canonicalizeThunkType(FT->getParamType(I), ParamAlign, in getThunkArgTypes()
233 Type *T = FT->getReturnType(); in getThunkRetType()
241 if (T->isVoidTy()) { in getThunkRetType()
242 if (FT->getNumParams()) { in getThunkRetType()
246 if (FT->getNumParams() > 1) { in getThunkRetType()
266 // FIXME: Sanity-check the sret type; if it's an integer or pointer, in getThunkRetType()
278 Arm64ArgTypes.push_back(FT->getParamType(0)); in getThunkRetType()
279 X64ArgTypes.push_back(FT->getParamType(0)); in getThunkRetType()
296 if (X64RetTy->isPointerTy()) { in getThunkRetType()
315 llvm::Type::getIntNTy(M->getContext(), SizeInBytes * 8), in canonicalizeThunkType()
324 if (T->isFloatTy()) { in canonicalizeThunkType()
329 if (T->isDoubleTy()) { in canonicalizeThunkType()
334 if (T->isFloatingPointTy()) { in canonicalizeThunkType()
339 auto &DL = M->getDataLayout(); in canonicalizeThunkType()
342 if (StructTy->getNumElements() == 1) in canonicalizeThunkType()
343 T = StructTy->getElementType(0); in canonicalizeThunkType()
345 if (T->isArrayTy()) { in canonicalizeThunkType()
346 Type *ElementTy = T->getArrayElementType(); in canonicalizeThunkType()
347 uint64_t ElementCnt = T->getArrayNumElements(); in canonicalizeThunkType()
350 if (ElementTy->isFloatTy() || ElementTy->isDoubleTy()) { in canonicalizeThunkType()
351 Out << (ElementTy->isFloatTy() ? "F" : "D") << TotalSizeBytes; in canonicalizeThunkType()
362 } else if (T->isFloatingPointTy()) { in canonicalizeThunkType()
368 if ((T->isIntegerTy() || T->isPointerTy()) && DL.getTypeSizeInBits(T) <= 64) { in canonicalizeThunkType()
401 if (Function *F = M->getFunction(ExitThunkName)) in buildExitThunk()
406 F->setCallingConv(CallingConv::ARM64EC_Thunk_Native); in buildExitThunk()
407 F->setSection(".wowthk$aa"); in buildExitThunk()
408 F->setComdat(M->getOrInsertComdat(ExitThunkName)); in buildExitThunk()
410 F->addFnAttr("frame-pointer", "all"); in buildExitThunk()
414 if (FT->getNumParams()) { in buildExitThunk()
418 F->addParamAttr(1, SRet); in buildExitThunk()
422 BasicBlock *BB = BasicBlock::Create(M->getContext(), "", F); in buildExitThunk()
425 M->getOrInsertGlobal("__os_arm64x_dispatch_call_no_redirect", PtrTy); in buildExitThunk()
427 auto &DL = M->getDataLayout(); in buildExitThunk()
432 Args.push_back(F->arg_begin()); in buildExitThunk()
434 Type *RetTy = Arm64Ty->getReturnType(); in buildExitThunk()
435 if (RetTy != X64Ty->getReturnType()) { in buildExitThunk()
446 make_range(F->arg_begin() + 1, F->arg_end()), in buildExitThunk()
447 make_range(X64Ty->param_begin() + X64TyOffset, X64Ty->param_end()), in buildExitThunk()
476 assert(Args.back()->getType() == X64ArgType); in buildExitThunk()
482 Call->setCallingConv(CallingConv::ARM64EC_Thunk_X64); in buildExitThunk()
485 if (RetTy != X64Ty->getReturnType()) { in buildExitThunk()
497 if (RetTy->isVoidTy()) in buildExitThunk()
511 getThunkType(F->getFunctionType(), F->getAttributes(), in buildEntryThunk()
514 if (Function *F = M->getFunction(EntryThunkName)) in buildEntryThunk()
519 Thunk->setCallingConv(CallingConv::ARM64EC_Thunk_X64); in buildEntryThunk()
520 Thunk->setSection(".wowthk$aa"); in buildEntryThunk()
521 Thunk->setComdat(M->getOrInsertComdat(EntryThunkName)); in buildEntryThunk()
523 Thunk->addFnAttr("frame-pointer", "all"); in buildEntryThunk()
525 BasicBlock *BB = BasicBlock::Create(M->getContext(), "", Thunk); in buildEntryThunk()
528 Type *RetTy = Arm64Ty->getReturnType(); in buildEntryThunk()
529 Type *X64RetType = X64Ty->getReturnType(); in buildEntryThunk()
531 bool TransformDirectToSRet = X64RetType->isVoidTy() && !RetTy->isVoidTy(); in buildEntryThunk()
534 (F->isVarArg() ? 5 : Thunk->arg_size()) - ThunkArgOffset; in buildEntryThunk()
535 assert(ArgTranslations.size() == (F->isVarArg() ? 5 : PassthroughArgSize)); in buildEntryThunk()
540 Value *Arg = Thunk->getArg(i + ThunkArgOffset); in buildEntryThunk()
541 Type *ArgTy = Arm64Ty->getParamType(i); in buildEntryThunk()
554 assert(Arg->getType() == ArgTy); in buildEntryThunk()
558 if (F->isVarArg()) { in buildEntryThunk()
565 Thunk->addParamAttr(5, Attribute::InReg); in buildEntryThunk()
566 Value *Arg = Thunk->getArg(5); in buildEntryThunk()
575 Value *Callee = Thunk->getArg(0); in buildEntryThunk()
579 auto SRetAttr = F->getAttributes().getParamAttr(0, Attribute::StructRet); in buildEntryThunk()
580 auto InRegAttr = F->getAttributes().getParamAttr(0, Attribute::InReg); in buildEntryThunk()
582 Thunk->addParamAttr(1, SRetAttr); in buildEntryThunk()
583 Call->addParamAttr(0, SRetAttr); in buildEntryThunk()
588 IRB.CreateStore(RetVal, IRB.CreateBitCast(Thunk->getArg(1), PtrTy)); in buildEntryThunk()
599 if (X64RetType->isVoidTy()) in buildEntryThunk()
607 // Builds the "guest exit thunk", a helper to call a function which may or may
608 // not be an exit thunk. (We optimistically assume non-dllimport function
615 getThunkType(F->getFunctionType(), F->getAttributes(), in buildGuestExitThunk()
618 auto MangledName = getArm64ECMangledFunctionName(F->getName().str()); in buildGuestExitThunk()
619 assert(MangledName && "Can't guest exit to function that's already native"); in buildGuestExitThunk()
628 GuestExit->setComdat(M->getOrInsertComdat(ThunkName)); in buildGuestExitThunk()
629 GuestExit->setSection(".wowthk$aa"); in buildGuestExitThunk()
630 GuestExit->setMetadata( in buildGuestExitThunk()
632 MDNode::get(M->getContext(), in buildGuestExitThunk()
633 MDString::get(M->getContext(), F->getName()))); in buildGuestExitThunk()
634 GuestExit->setMetadata( in buildGuestExitThunk()
636 MDNode::get(M->getContext(), in buildGuestExitThunk()
637 MDString::get(M->getContext(), *MangledName))); in buildGuestExitThunk()
638 F->setMetadata("arm64ec_hasguestexit", MDNode::get(M->getContext(), {})); in buildGuestExitThunk()
639 BasicBlock *BB = BasicBlock::Create(M->getContext(), "", GuestExit); in buildGuestExitThunk()
644 if (cfguard_module_flag == 2 && !F->hasFnAttribute("guard_nocf")) in buildGuestExitThunk()
652 Function *Thunk = buildExitThunk(F->getFunctionType(), F->getAttributes()); in buildGuestExitThunk()
658 GuardCheck->setCallingConv(CallingConv::CFGuard_Check); in buildGuestExitThunk()
662 for (Argument &Arg : GuestExit->args()) in buildGuestExitThunk()
665 Call->setTailCallKind(llvm::CallInst::TCK_MustTail); in buildGuestExitThunk()
667 if (Call->getType()->isVoidTy()) in buildGuestExitThunk()
672 auto SRetAttr = F->getAttributes().getParamAttr(0, Attribute::StructRet); in buildGuestExitThunk()
673 auto InRegAttr = F->getAttributes().getParamAttr(0, Attribute::InReg); in buildGuestExitThunk()
675 GuestExit->addParamAttr(0, SRetAttr); in buildGuestExitThunk()
676 Call->addParamAttr(0, SRetAttr); in buildGuestExitThunk()
687 Function *F = cast<Function>(MangledAlias->getAliasee()); in buildPatchableThunk()
689 getThunkType(F->getFunctionType(), F->getAttributes(), in buildPatchableThunk()
692 std::string ThunkName(MangledAlias->getName()); in buildPatchableThunk()
701 GuestExit->setComdat(M->getOrInsertComdat(ThunkName)); in buildPatchableThunk()
702 GuestExit->setSection(".wowthk$aa"); in buildPatchableThunk()
703 BasicBlock *BB = BasicBlock::Create(M->getContext(), "", GuestExit); in buildPatchableThunk()
711 buildExitThunk(F->getFunctionType(), F->getAttributes()); in buildPatchableThunk()
714 {UnmangledAlias, ExitThunk, UnmangledAlias->getAliasee()}); in buildPatchableThunk()
717 Dispatch->setCallingConv(CallingConv::CFGuard_Check); in buildPatchableThunk()
721 for (Argument &Arg : GuestExit->args()) in buildPatchableThunk()
724 Call->setTailCallKind(llvm::CallInst::TCK_MustTail); in buildPatchableThunk()
726 if (Call->getType()->isVoidTy()) in buildPatchableThunk()
731 auto SRetAttr = F->getAttributes().getParamAttr(0, Attribute::StructRet); in buildPatchableThunk()
732 auto InRegAttr = F->getAttributes().getParamAttr(0, Attribute::InReg); in buildPatchableThunk()
734 GuestExit->addParamAttr(0, SRetAttr); in buildPatchableThunk()
735 Call->addParamAttr(0, SRetAttr); in buildPatchableThunk()
738 MangledAlias->setAliasee(GuestExit); in buildPatchableThunk()
744 assert(Triple(CB->getModule()->getTargetTriple()).isOSWindows() && in lowerCall()
748 Value *CalledOperand = CB->getCalledOperand(); in lowerCall()
753 if (auto Bundle = CB->getOperandBundle(LLVMContext::OB_funclet)) in lowerCall()
758 if (cfguard_module_flag == 2 && !CB->hasFnAttr("guard_nocf")) in lowerCall()
766 Function *Thunk = buildExitThunk(CB->getFunctionType(), CB->getAttributes()); in lowerCall()
774 GuardCheck->setCallingConv(CallingConv::CFGuard_Check); in lowerCall()
776 Value *GuardRetVal = B.CreateBitCast(GuardCheck, CalledOperand->getType()); in lowerCall()
777 CB->setCalledOperand(GuardRetVal); in lowerCall()
788 mdconst::extract_or_null<ConstantInt>(M->getModuleFlag("cfguard"))) in runOnModule()
789 cfguard_module_flag = MD->getZExtValue(); in runOnModule()
791 PtrTy = PointerType::getUnqual(M->getContext()); in runOnModule()
792 I64Ty = Type::getInt64Ty(M->getContext()); in runOnModule()
793 VoidTy = Type::getVoidTy(M->getContext()); in runOnModule()
800 M->getOrInsertGlobal("__os_arm64x_check_icall_cfg", GuardFnPtrType); in runOnModule()
802 M->getOrInsertGlobal("__os_arm64x_check_icall", GuardFnPtrType); in runOnModule()
804 M->getOrInsertGlobal("__os_arm64x_dispatch_call", DispatchFnPtrType); in runOnModule()
831 MDNode::get(M->getContext(), in runOnModule()
832 MDString::get(M->getContext(), in runOnModule()
834 A->setAliasee(&F); in runOnModule()
837 A->setDLLStorageClass(GlobalValue::DLLExportStorageClass); in runOnModule()
872 auto F = dyn_cast<Function>(GA ? GA->getAliasee() : O); in runOnModule()
874 {O, buildExitThunk(F->getFunctionType(), F->getAttributes()), in runOnModule()
876 if (!GA && !F->hasDLLImportStorageClass()) in runOnModule()
891 ConstantInt::get(M->getContext(), APInt(32, uint8_t(Thunk.Kind)))})); in runOnModule()
894 llvm::ArrayType::get(ThunkMappingArrayElems[0]->getType(), in runOnModule()
897 new GlobalVariable(Mod, ThunkMappingArray->getType(), /*isConstant*/ false, in runOnModule()
921 MDNode::get(M->getContext(), in processFunction()
922 MDString::get(M->getContext(), F.getName()))); in processFunction()
923 if (F.hasComdat() && F.getComdat()->getName() == F.getName()) { in processFunction()
924 Comdat *MangledComdat = M->getOrInsertComdat(MangledName.value()); in processFunction()
926 to_vector(F.getComdat()->getUsers()); in processFunction()
928 User->setComdat(MangledComdat); in processFunction()
941 if (!CB || CB->getCallingConv() == CallingConv::ARM64EC_Thunk_X64 || in processFunction()
942 CB->isInlineAsm()) in processFunction()
950 if (Function *F = CB->getCalledFunction()) { in processFunction()
951 if (!LowerDirectToIndirect || F->hasLocalLinkage() || in processFunction()
952 F->isIntrinsic() || !F->isDeclaration()) in processFunction()
960 if (GlobalAlias *A = dyn_cast<GlobalAlias>(CB->getCalledOperand())) { in processFunction()
963 CB->setCalledOperand(I->second); in processFunction()
964 DirectCalledFns.insert(I->first); in processFunction()