1 //===----- EPCGenericRTDyldMemoryManager.cpp - EPC-bbasde MemMgr -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h"
10 #include "llvm/ExecutionEngine/Orc/EPCGenericMemoryAccess.h"
11 #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
12 #include "llvm/Support/Alignment.h"
13 #include "llvm/Support/FormatVariadic.h"
14
15 #define DEBUG_TYPE "orc"
16
17 using namespace llvm::orc::shared;
18
19 namespace llvm {
20 namespace orc {
21
22 Expected<std::unique_ptr<EPCGenericRTDyldMemoryManager>>
CreateWithDefaultBootstrapSymbols(ExecutorProcessControl & EPC)23 EPCGenericRTDyldMemoryManager::CreateWithDefaultBootstrapSymbols(
24 ExecutorProcessControl &EPC) {
25 SymbolAddrs SAs;
26 if (auto Err = EPC.getBootstrapSymbols(
27 {{SAs.Instance, rt::SimpleExecutorMemoryManagerInstanceName},
28 {SAs.Reserve, rt::SimpleExecutorMemoryManagerReserveWrapperName},
29 {SAs.Finalize, rt::SimpleExecutorMemoryManagerFinalizeWrapperName},
30 {SAs.Deallocate,
31 rt::SimpleExecutorMemoryManagerDeallocateWrapperName},
32 {SAs.RegisterEHFrame, rt::RegisterEHFrameSectionWrapperName},
33 {SAs.DeregisterEHFrame, rt::DeregisterEHFrameSectionWrapperName}}))
34 return std::move(Err);
35 return std::make_unique<EPCGenericRTDyldMemoryManager>(EPC, std::move(SAs));
36 }
37
EPCGenericRTDyldMemoryManager(ExecutorProcessControl & EPC,SymbolAddrs SAs)38 EPCGenericRTDyldMemoryManager::EPCGenericRTDyldMemoryManager(
39 ExecutorProcessControl &EPC, SymbolAddrs SAs)
40 : EPC(EPC), SAs(std::move(SAs)) {
41 LLVM_DEBUG(dbgs() << "Created remote allocator " << (void *)this << "\n");
42 }
43
~EPCGenericRTDyldMemoryManager()44 EPCGenericRTDyldMemoryManager::~EPCGenericRTDyldMemoryManager() {
45 LLVM_DEBUG(dbgs() << "Destroyed remote allocator " << (void *)this << "\n");
46 if (!ErrMsg.empty())
47 errs() << "Destroying with existing errors:\n" << ErrMsg << "\n";
48
49 Error Err = Error::success();
50 if (auto Err2 = EPC.callSPSWrapper<
51 rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>(
52 SAs.Reserve, Err, SAs.Instance, FinalizedAllocs)) {
53 // FIXME: Report errors through EPC once that functionality is available.
54 logAllUnhandledErrors(std::move(Err2), errs(), "");
55 return;
56 }
57
58 if (Err)
59 logAllUnhandledErrors(std::move(Err), errs(), "");
60 }
61
allocateCodeSection(uintptr_t Size,unsigned Alignment,unsigned SectionID,StringRef SectionName)62 uint8_t *EPCGenericRTDyldMemoryManager::allocateCodeSection(
63 uintptr_t Size, unsigned Alignment, unsigned SectionID,
64 StringRef SectionName) {
65 std::lock_guard<std::mutex> Lock(M);
66 LLVM_DEBUG({
67 dbgs() << "Allocator " << (void *)this << " allocating code section "
68 << SectionName << ": size = " << formatv("{0:x}", Size)
69 << " bytes, alignment = " << Alignment << "\n";
70 });
71 auto &Seg = Unmapped.back().CodeAllocs;
72 Seg.emplace_back(Size, Alignment);
73 return reinterpret_cast<uint8_t *>(
74 alignAddr(Seg.back().Contents.get(), Align(Alignment)));
75 }
76
allocateDataSection(uintptr_t Size,unsigned Alignment,unsigned SectionID,StringRef SectionName,bool IsReadOnly)77 uint8_t *EPCGenericRTDyldMemoryManager::allocateDataSection(
78 uintptr_t Size, unsigned Alignment, unsigned SectionID,
79 StringRef SectionName, bool IsReadOnly) {
80 std::lock_guard<std::mutex> Lock(M);
81 LLVM_DEBUG({
82 dbgs() << "Allocator " << (void *)this << " allocating "
83 << (IsReadOnly ? "ro" : "rw") << "-data section " << SectionName
84 << ": size = " << formatv("{0:x}", Size) << " bytes, alignment "
85 << Alignment << ")\n";
86 });
87
88 auto &Seg =
89 IsReadOnly ? Unmapped.back().RODataAllocs : Unmapped.back().RWDataAllocs;
90
91 Seg.emplace_back(Size, Alignment);
92 return reinterpret_cast<uint8_t *>(
93 alignAddr(Seg.back().Contents.get(), Align(Alignment)));
94 }
95
reserveAllocationSpace(uintptr_t CodeSize,Align CodeAlign,uintptr_t RODataSize,Align RODataAlign,uintptr_t RWDataSize,Align RWDataAlign)96 void EPCGenericRTDyldMemoryManager::reserveAllocationSpace(
97 uintptr_t CodeSize, Align CodeAlign, uintptr_t RODataSize,
98 Align RODataAlign, uintptr_t RWDataSize, Align RWDataAlign) {
99
100 {
101 std::lock_guard<std::mutex> Lock(M);
102 // If there's already an error then bail out.
103 if (!ErrMsg.empty())
104 return;
105
106 if (CodeAlign > EPC.getPageSize()) {
107 ErrMsg = "Invalid code alignment in reserveAllocationSpace";
108 return;
109 }
110 if (RODataAlign > EPC.getPageSize()) {
111 ErrMsg = "Invalid ro-data alignment in reserveAllocationSpace";
112 return;
113 }
114 if (RWDataAlign > EPC.getPageSize()) {
115 ErrMsg = "Invalid rw-data alignment in reserveAllocationSpace";
116 return;
117 }
118 }
119
120 uint64_t TotalSize = 0;
121 TotalSize += alignTo(CodeSize, EPC.getPageSize());
122 TotalSize += alignTo(RODataSize, EPC.getPageSize());
123 TotalSize += alignTo(RWDataSize, EPC.getPageSize());
124
125 LLVM_DEBUG({
126 dbgs() << "Allocator " << (void *)this << " reserving "
127 << formatv("{0:x}", TotalSize) << " bytes.\n";
128 });
129
130 Expected<ExecutorAddr> TargetAllocAddr((ExecutorAddr()));
131 if (auto Err = EPC.callSPSWrapper<
132 rt::SPSSimpleExecutorMemoryManagerReserveSignature>(
133 SAs.Reserve, TargetAllocAddr, SAs.Instance, TotalSize)) {
134 std::lock_guard<std::mutex> Lock(M);
135 ErrMsg = toString(std::move(Err));
136 return;
137 }
138 if (!TargetAllocAddr) {
139 std::lock_guard<std::mutex> Lock(M);
140 ErrMsg = toString(TargetAllocAddr.takeError());
141 return;
142 }
143
144 std::lock_guard<std::mutex> Lock(M);
145 Unmapped.push_back(SectionAllocGroup());
146 Unmapped.back().RemoteCode = {
147 *TargetAllocAddr, ExecutorAddrDiff(alignTo(CodeSize, EPC.getPageSize()))};
148 Unmapped.back().RemoteROData = {
149 Unmapped.back().RemoteCode.End,
150 ExecutorAddrDiff(alignTo(RODataSize, EPC.getPageSize()))};
151 Unmapped.back().RemoteRWData = {
152 Unmapped.back().RemoteROData.End,
153 ExecutorAddrDiff(alignTo(RWDataSize, EPC.getPageSize()))};
154 }
155
needsToReserveAllocationSpace()156 bool EPCGenericRTDyldMemoryManager::needsToReserveAllocationSpace() {
157 return true;
158 }
159
registerEHFrames(uint8_t * Addr,uint64_t LoadAddr,size_t Size)160 void EPCGenericRTDyldMemoryManager::registerEHFrames(uint8_t *Addr,
161 uint64_t LoadAddr,
162 size_t Size) {
163 LLVM_DEBUG({
164 dbgs() << "Allocator " << (void *)this << " added unfinalized eh-frame "
165 << formatv("[ {0:x} {1:x} ]", LoadAddr, LoadAddr + Size) << "\n";
166 });
167 std::lock_guard<std::mutex> Lock(M);
168 // Bail out early if there's already an error.
169 if (!ErrMsg.empty())
170 return;
171
172 ExecutorAddr LA(LoadAddr);
173 for (auto &SecAllocGroup : llvm::reverse(Unfinalized)) {
174 if (SecAllocGroup.RemoteCode.contains(LA) ||
175 SecAllocGroup.RemoteROData.contains(LA) ||
176 SecAllocGroup.RemoteRWData.contains(LA)) {
177 SecAllocGroup.UnfinalizedEHFrames.push_back({LA, Size});
178 return;
179 }
180 }
181 ErrMsg = "eh-frame does not lie inside unfinalized alloc";
182 }
183
deregisterEHFrames()184 void EPCGenericRTDyldMemoryManager::deregisterEHFrames() {
185 // This is a no-op for us: We've registered a deallocation action for it.
186 }
187
notifyObjectLoaded(RuntimeDyld & Dyld,const object::ObjectFile & Obj)188 void EPCGenericRTDyldMemoryManager::notifyObjectLoaded(
189 RuntimeDyld &Dyld, const object::ObjectFile &Obj) {
190 std::lock_guard<std::mutex> Lock(M);
191 LLVM_DEBUG(dbgs() << "Allocator " << (void *)this << " applied mappings:\n");
192 for (auto &ObjAllocs : Unmapped) {
193 mapAllocsToRemoteAddrs(Dyld, ObjAllocs.CodeAllocs,
194 ObjAllocs.RemoteCode.Start);
195 mapAllocsToRemoteAddrs(Dyld, ObjAllocs.RODataAllocs,
196 ObjAllocs.RemoteROData.Start);
197 mapAllocsToRemoteAddrs(Dyld, ObjAllocs.RWDataAllocs,
198 ObjAllocs.RemoteRWData.Start);
199 Unfinalized.push_back(std::move(ObjAllocs));
200 }
201 Unmapped.clear();
202 }
203
finalizeMemory(std::string * ErrMsg)204 bool EPCGenericRTDyldMemoryManager::finalizeMemory(std::string *ErrMsg) {
205 LLVM_DEBUG(dbgs() << "Allocator " << (void *)this << " finalizing:\n");
206
207 // If there's an error then bail out here.
208 std::vector<SectionAllocGroup> SecAllocGroups;
209 {
210 std::lock_guard<std::mutex> Lock(M);
211 if (ErrMsg && !this->ErrMsg.empty()) {
212 *ErrMsg = std::move(this->ErrMsg);
213 return true;
214 }
215 std::swap(SecAllocGroups, Unfinalized);
216 }
217
218 // Loop over unfinalized objects to make finalization requests.
219 for (auto &SecAllocGroup : SecAllocGroups) {
220
221 MemProt SegMemProts[3] = {MemProt::Read | MemProt::Exec, MemProt::Read,
222 MemProt::Read | MemProt::Write};
223
224 ExecutorAddrRange *RemoteAddrs[3] = {&SecAllocGroup.RemoteCode,
225 &SecAllocGroup.RemoteROData,
226 &SecAllocGroup.RemoteRWData};
227
228 std::vector<SectionAlloc> *SegSections[3] = {&SecAllocGroup.CodeAllocs,
229 &SecAllocGroup.RODataAllocs,
230 &SecAllocGroup.RWDataAllocs};
231
232 tpctypes::FinalizeRequest FR;
233 std::unique_ptr<char[]> AggregateContents[3];
234
235 for (unsigned I = 0; I != 3; ++I) {
236 FR.Segments.push_back({});
237 auto &Seg = FR.Segments.back();
238 Seg.RAG = SegMemProts[I];
239 Seg.Addr = RemoteAddrs[I]->Start;
240 for (auto &SecAlloc : *SegSections[I]) {
241 Seg.Size = alignTo(Seg.Size, SecAlloc.Align);
242 Seg.Size += SecAlloc.Size;
243 }
244 AggregateContents[I] = std::make_unique<char[]>(Seg.Size);
245 size_t SecOffset = 0;
246 for (auto &SecAlloc : *SegSections[I]) {
247 SecOffset = alignTo(SecOffset, SecAlloc.Align);
248 memcpy(&AggregateContents[I][SecOffset],
249 reinterpret_cast<const char *>(
250 alignAddr(SecAlloc.Contents.get(), Align(SecAlloc.Align))),
251 SecAlloc.Size);
252 SecOffset += SecAlloc.Size;
253 // FIXME: Can we reset SecAlloc.Content here, now that it's copied into
254 // the aggregated content?
255 }
256 Seg.Content = {AggregateContents[I].get(), SecOffset};
257 }
258
259 for (auto &Frame : SecAllocGroup.UnfinalizedEHFrames)
260 FR.Actions.push_back(
261 {cantFail(
262 WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>(
263 SAs.RegisterEHFrame, Frame)),
264 cantFail(
265 WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>(
266 SAs.DeregisterEHFrame, Frame))});
267
268 // We'll also need to make an extra allocation for the eh-frame wrapper call
269 // arguments.
270 Error FinalizeErr = Error::success();
271 if (auto Err = EPC.callSPSWrapper<
272 rt::SPSSimpleExecutorMemoryManagerFinalizeSignature>(
273 SAs.Finalize, FinalizeErr, SAs.Instance, std::move(FR))) {
274 std::lock_guard<std::mutex> Lock(M);
275 this->ErrMsg = toString(std::move(Err));
276 dbgs() << "Serialization error: " << this->ErrMsg << "\n";
277 if (ErrMsg)
278 *ErrMsg = this->ErrMsg;
279 return true;
280 }
281 if (FinalizeErr) {
282 std::lock_guard<std::mutex> Lock(M);
283 this->ErrMsg = toString(std::move(FinalizeErr));
284 dbgs() << "Finalization error: " << this->ErrMsg << "\n";
285 if (ErrMsg)
286 *ErrMsg = this->ErrMsg;
287 return true;
288 }
289 }
290
291 return false;
292 }
293
mapAllocsToRemoteAddrs(RuntimeDyld & Dyld,std::vector<SectionAlloc> & Allocs,ExecutorAddr NextAddr)294 void EPCGenericRTDyldMemoryManager::mapAllocsToRemoteAddrs(
295 RuntimeDyld &Dyld, std::vector<SectionAlloc> &Allocs,
296 ExecutorAddr NextAddr) {
297 for (auto &Alloc : Allocs) {
298 NextAddr.setValue(alignTo(NextAddr.getValue(), Alloc.Align));
299 LLVM_DEBUG({
300 dbgs() << " " << static_cast<void *>(Alloc.Contents.get()) << " -> "
301 << format("0x%016" PRIx64, NextAddr.getValue()) << "\n";
302 });
303 Dyld.mapSectionAddress(reinterpret_cast<const void *>(alignAddr(
304 Alloc.Contents.get(), Align(Alloc.Align))),
305 NextAddr.getValue());
306 Alloc.RemoteAddr = NextAddr;
307 // Only advance NextAddr if it was non-null to begin with,
308 // otherwise leave it as null.
309 if (NextAddr)
310 NextAddr += ExecutorAddrDiff(Alloc.Size);
311 }
312 }
313
314 } // end namespace orc
315 } // end namespace llvm
316