1 //===--------- SMEABI - SME ABI-------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass implements parts of the the SME ABI, such as:
10 // * Using the lazy-save mechanism before enabling the use of ZA.
11 // * Setting up the lazy-save mechanism around invokes.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "AArch64.h"
16 #include "Utils/AArch64BaseInfo.h"
17 #include "Utils/AArch64SMEAttributes.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/IR/Constants.h"
20 #include "llvm/IR/IRBuilder.h"
21 #include "llvm/IR/Instructions.h"
22 #include "llvm/IR/IntrinsicInst.h"
23 #include "llvm/IR/IntrinsicsAArch64.h"
24 #include "llvm/IR/LLVMContext.h"
25 #include "llvm/IR/Module.h"
26 #include "llvm/InitializePasses.h"
27 #include "llvm/Support/Debug.h"
28 #include "llvm/Transforms/Utils/Cloning.h"
29
30 using namespace llvm;
31
32 #define DEBUG_TYPE "aarch64-sme-abi"
33
34 namespace {
35 struct SMEABI : public FunctionPass {
36 static char ID; // Pass identification, replacement for typeid
SMEABI__anon6b3721c60111::SMEABI37 SMEABI() : FunctionPass(ID) {
38 initializeSMEABIPass(*PassRegistry::getPassRegistry());
39 }
40
41 bool runOnFunction(Function &F) override;
42
43 private:
44 bool updateNewStateFunctions(Module *M, Function *F, IRBuilder<> &Builder,
45 SMEAttrs FnAttrs);
46 };
47 } // end anonymous namespace
48
49 char SMEABI::ID = 0;
50 static const char *name = "SME ABI Pass";
INITIALIZE_PASS_BEGIN(SMEABI,DEBUG_TYPE,name,false,false)51 INITIALIZE_PASS_BEGIN(SMEABI, DEBUG_TYPE, name, false, false)
52 INITIALIZE_PASS_END(SMEABI, DEBUG_TYPE, name, false, false)
53
54 FunctionPass *llvm::createSMEABIPass() { return new SMEABI(); }
55
56 //===----------------------------------------------------------------------===//
57 // Utility functions
58 //===----------------------------------------------------------------------===//
59
60 // Utility function to emit a call to __arm_tpidr2_save and clear TPIDR2_EL0.
emitTPIDR2Save(Module * M,IRBuilder<> & Builder)61 void emitTPIDR2Save(Module *M, IRBuilder<> &Builder) {
62 auto *TPIDR2SaveTy =
63 FunctionType::get(Builder.getVoidTy(), {}, /*IsVarArgs=*/false);
64 auto Attrs = AttributeList().addFnAttribute(M->getContext(),
65 "aarch64_pstate_sm_compatible");
66 FunctionCallee Callee =
67 M->getOrInsertFunction("__arm_tpidr2_save", TPIDR2SaveTy, Attrs);
68 CallInst *Call = Builder.CreateCall(Callee);
69 Call->setCallingConv(
70 CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0);
71
72 // A save to TPIDR2 should be followed by clearing TPIDR2_EL0.
73 Function *WriteIntr =
74 Intrinsic::getDeclaration(M, Intrinsic::aarch64_sme_set_tpidr2);
75 Builder.CreateCall(WriteIntr->getFunctionType(), WriteIntr,
76 Builder.getInt64(0));
77 }
78
79 /// This function generates code at the beginning and end of a function marked
80 /// with either `aarch64_new_za` or `aarch64_new_zt0`.
81 /// At the beginning of the function, the following code is generated:
82 /// - Commit lazy-save if active [Private-ZA Interface*]
83 /// - Enable PSTATE.ZA [Private-ZA Interface]
84 /// - Zero ZA [Has New ZA State]
85 /// - Zero ZT0 [Has New ZT0 State]
86 ///
87 /// * A function with new ZT0 state will not change ZA, so committing the
88 /// lazy-save is not strictly necessary. However, the lazy-save mechanism
89 /// may be active on entry to the function, with PSTATE.ZA set to 1. If
90 /// the new ZT0 function calls a function that does not share ZT0, we will
91 /// need to conditionally SMSTOP ZA before the call, setting PSTATE.ZA to 0.
92 /// For this reason, it's easier to always commit the lazy-save at the
93 /// beginning of the function regardless of whether it has ZA state.
94 ///
95 /// At the end of the function, PSTATE.ZA is disabled if the function has a
96 /// Private-ZA Interface. A function is considered to have a Private-ZA
97 /// interface if it does not share ZA or ZT0.
98 ///
updateNewStateFunctions(Module * M,Function * F,IRBuilder<> & Builder,SMEAttrs FnAttrs)99 bool SMEABI::updateNewStateFunctions(Module *M, Function *F,
100 IRBuilder<> &Builder, SMEAttrs FnAttrs) {
101 LLVMContext &Context = F->getContext();
102 BasicBlock *OrigBB = &F->getEntryBlock();
103 Builder.SetInsertPoint(&OrigBB->front());
104
105 // Commit any active lazy-saves if this is a Private-ZA function. If the
106 // value read from TPIDR2_EL0 is not null on entry to the function then
107 // the lazy-saving scheme is active and we should call __arm_tpidr2_save
108 // to commit the lazy save.
109 if (FnAttrs.hasPrivateZAInterface()) {
110 // Create the new blocks for reading TPIDR2_EL0 & enabling ZA state.
111 auto *SaveBB = OrigBB->splitBasicBlock(OrigBB->begin(), "save.za", true);
112 auto *PreludeBB = BasicBlock::Create(Context, "prelude", F, SaveBB);
113
114 // Read TPIDR2_EL0 in PreludeBB & branch to SaveBB if not 0.
115 Builder.SetInsertPoint(PreludeBB);
116 Function *TPIDR2Intr =
117 Intrinsic::getDeclaration(M, Intrinsic::aarch64_sme_get_tpidr2);
118 auto *TPIDR2 = Builder.CreateCall(TPIDR2Intr->getFunctionType(), TPIDR2Intr,
119 {}, "tpidr2");
120 auto *Cmp = Builder.CreateCmp(ICmpInst::ICMP_NE, TPIDR2,
121 Builder.getInt64(0), "cmp");
122 Builder.CreateCondBr(Cmp, SaveBB, OrigBB);
123
124 // Create a call __arm_tpidr2_save, which commits the lazy save.
125 Builder.SetInsertPoint(&SaveBB->back());
126 emitTPIDR2Save(M, Builder);
127
128 // Enable pstate.za at the start of the function.
129 Builder.SetInsertPoint(&OrigBB->front());
130 Function *EnableZAIntr =
131 Intrinsic::getDeclaration(M, Intrinsic::aarch64_sme_za_enable);
132 Builder.CreateCall(EnableZAIntr->getFunctionType(), EnableZAIntr);
133 }
134
135 if (FnAttrs.isNewZA()) {
136 Function *ZeroIntr =
137 Intrinsic::getDeclaration(M, Intrinsic::aarch64_sme_zero);
138 Builder.CreateCall(ZeroIntr->getFunctionType(), ZeroIntr,
139 Builder.getInt32(0xff));
140 }
141
142 if (FnAttrs.isNewZT0()) {
143 Function *ClearZT0Intr =
144 Intrinsic::getDeclaration(M, Intrinsic::aarch64_sme_zero_zt);
145 Builder.CreateCall(ClearZT0Intr->getFunctionType(), ClearZT0Intr,
146 {Builder.getInt32(0)});
147 }
148
149 if (FnAttrs.hasPrivateZAInterface()) {
150 // Before returning, disable pstate.za
151 for (BasicBlock &BB : *F) {
152 Instruction *T = BB.getTerminator();
153 if (!T || !isa<ReturnInst>(T))
154 continue;
155 Builder.SetInsertPoint(T);
156 Function *DisableZAIntr =
157 Intrinsic::getDeclaration(M, Intrinsic::aarch64_sme_za_disable);
158 Builder.CreateCall(DisableZAIntr->getFunctionType(), DisableZAIntr);
159 }
160 }
161
162 F->addFnAttr("aarch64_expanded_pstate_za");
163 return true;
164 }
165
runOnFunction(Function & F)166 bool SMEABI::runOnFunction(Function &F) {
167 Module *M = F.getParent();
168 LLVMContext &Context = F.getContext();
169 IRBuilder<> Builder(Context);
170
171 if (F.isDeclaration() || F.hasFnAttribute("aarch64_expanded_pstate_za"))
172 return false;
173
174 bool Changed = false;
175 SMEAttrs FnAttrs(F);
176 if (FnAttrs.isNewZA() || FnAttrs.isNewZT0())
177 Changed |= updateNewStateFunctions(M, &F, Builder, FnAttrs);
178
179 return Changed;
180 }
181