1 //===----- EPCGenericRTDyldMemoryManager.cpp - EPC-bbasde MemMgr -----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h" 10 #include "llvm/ExecutionEngine/Orc/EPCGenericMemoryAccess.h" 11 #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" 12 #include "llvm/Support/Alignment.h" 13 #include "llvm/Support/FormatVariadic.h" 14 15 #define DEBUG_TYPE "orc" 16 17 using namespace llvm::orc::shared; 18 19 namespace llvm { 20 namespace orc { 21 22 Expected<std::unique_ptr<EPCGenericRTDyldMemoryManager>> 23 EPCGenericRTDyldMemoryManager::CreateWithDefaultBootstrapSymbols( 24 ExecutorProcessControl &EPC) { 25 SymbolAddrs SAs; 26 if (auto Err = EPC.getBootstrapSymbols( 27 {{SAs.Instance, rt::SimpleExecutorMemoryManagerInstanceName}, 28 {SAs.Reserve, rt::SimpleExecutorMemoryManagerReserveWrapperName}, 29 {SAs.Finalize, rt::SimpleExecutorMemoryManagerFinalizeWrapperName}, 30 {SAs.Deallocate, 31 rt::SimpleExecutorMemoryManagerDeallocateWrapperName}, 32 {SAs.RegisterEHFrame, rt::RegisterEHFrameSectionWrapperName}, 33 {SAs.DeregisterEHFrame, rt::DeregisterEHFrameSectionWrapperName}})) 34 return std::move(Err); 35 return std::make_unique<EPCGenericRTDyldMemoryManager>(EPC, std::move(SAs)); 36 } 37 38 EPCGenericRTDyldMemoryManager::EPCGenericRTDyldMemoryManager( 39 ExecutorProcessControl &EPC, SymbolAddrs SAs) 40 : EPC(EPC), SAs(std::move(SAs)) { 41 LLVM_DEBUG(dbgs() << "Created remote allocator " << (void *)this << "\n"); 42 } 43 44 EPCGenericRTDyldMemoryManager::~EPCGenericRTDyldMemoryManager() { 45 LLVM_DEBUG(dbgs() << "Destroyed remote allocator " << (void *)this << "\n"); 46 if (!ErrMsg.empty()) 47 errs() << "Destroying with existing errors:\n" << ErrMsg << "\n"; 48 49 Error Err = Error::success(); 50 if (auto Err2 = EPC.callSPSWrapper< 51 rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>( 52 SAs.Reserve, Err, SAs.Instance, FinalizedAllocs)) { 53 // FIXME: Report errors through EPC once that functionality is available. 54 logAllUnhandledErrors(std::move(Err2), errs(), ""); 55 return; 56 } 57 58 if (Err) 59 logAllUnhandledErrors(std::move(Err), errs(), ""); 60 } 61 62 uint8_t *EPCGenericRTDyldMemoryManager::allocateCodeSection( 63 uintptr_t Size, unsigned Alignment, unsigned SectionID, 64 StringRef SectionName) { 65 std::lock_guard<std::mutex> Lock(M); 66 LLVM_DEBUG({ 67 dbgs() << "Allocator " << (void *)this << " allocating code section " 68 << SectionName << ": size = " << formatv("{0:x}", Size) 69 << " bytes, alignment = " << Alignment << "\n"; 70 }); 71 auto &Seg = Unmapped.back().CodeAllocs; 72 Seg.emplace_back(Size, Alignment); 73 return reinterpret_cast<uint8_t *>( 74 alignAddr(Seg.back().Contents.get(), Align(Alignment))); 75 } 76 77 uint8_t *EPCGenericRTDyldMemoryManager::allocateDataSection( 78 uintptr_t Size, unsigned Alignment, unsigned SectionID, 79 StringRef SectionName, bool IsReadOnly) { 80 std::lock_guard<std::mutex> Lock(M); 81 LLVM_DEBUG({ 82 dbgs() << "Allocator " << (void *)this << " allocating " 83 << (IsReadOnly ? "ro" : "rw") << "-data section " << SectionName 84 << ": size = " << formatv("{0:x}", Size) << " bytes, alignment " 85 << Alignment << ")\n"; 86 }); 87 88 auto &Seg = 89 IsReadOnly ? Unmapped.back().RODataAllocs : Unmapped.back().RWDataAllocs; 90 91 Seg.emplace_back(Size, Alignment); 92 return reinterpret_cast<uint8_t *>( 93 alignAddr(Seg.back().Contents.get(), Align(Alignment))); 94 } 95 96 void EPCGenericRTDyldMemoryManager::reserveAllocationSpace( 97 uintptr_t CodeSize, Align CodeAlign, uintptr_t RODataSize, 98 Align RODataAlign, uintptr_t RWDataSize, Align RWDataAlign) { 99 100 { 101 std::lock_guard<std::mutex> Lock(M); 102 // If there's already an error then bail out. 103 if (!ErrMsg.empty()) 104 return; 105 106 if (CodeAlign > EPC.getPageSize()) { 107 ErrMsg = "Invalid code alignment in reserveAllocationSpace"; 108 return; 109 } 110 if (RODataAlign > EPC.getPageSize()) { 111 ErrMsg = "Invalid ro-data alignment in reserveAllocationSpace"; 112 return; 113 } 114 if (RWDataAlign > EPC.getPageSize()) { 115 ErrMsg = "Invalid rw-data alignment in reserveAllocationSpace"; 116 return; 117 } 118 } 119 120 uint64_t TotalSize = 0; 121 TotalSize += alignTo(CodeSize, EPC.getPageSize()); 122 TotalSize += alignTo(RODataSize, EPC.getPageSize()); 123 TotalSize += alignTo(RWDataSize, EPC.getPageSize()); 124 125 LLVM_DEBUG({ 126 dbgs() << "Allocator " << (void *)this << " reserving " 127 << formatv("{0:x}", TotalSize) << " bytes.\n"; 128 }); 129 130 Expected<ExecutorAddr> TargetAllocAddr((ExecutorAddr())); 131 if (auto Err = EPC.callSPSWrapper< 132 rt::SPSSimpleExecutorMemoryManagerReserveSignature>( 133 SAs.Reserve, TargetAllocAddr, SAs.Instance, TotalSize)) { 134 std::lock_guard<std::mutex> Lock(M); 135 ErrMsg = toString(std::move(Err)); 136 return; 137 } 138 if (!TargetAllocAddr) { 139 std::lock_guard<std::mutex> Lock(M); 140 ErrMsg = toString(TargetAllocAddr.takeError()); 141 return; 142 } 143 144 std::lock_guard<std::mutex> Lock(M); 145 Unmapped.push_back(SectionAllocGroup()); 146 Unmapped.back().RemoteCode = { 147 *TargetAllocAddr, ExecutorAddrDiff(alignTo(CodeSize, EPC.getPageSize()))}; 148 Unmapped.back().RemoteROData = { 149 Unmapped.back().RemoteCode.End, 150 ExecutorAddrDiff(alignTo(RODataSize, EPC.getPageSize()))}; 151 Unmapped.back().RemoteRWData = { 152 Unmapped.back().RemoteROData.End, 153 ExecutorAddrDiff(alignTo(RWDataSize, EPC.getPageSize()))}; 154 } 155 156 bool EPCGenericRTDyldMemoryManager::needsToReserveAllocationSpace() { 157 return true; 158 } 159 160 void EPCGenericRTDyldMemoryManager::registerEHFrames(uint8_t *Addr, 161 uint64_t LoadAddr, 162 size_t Size) { 163 LLVM_DEBUG({ 164 dbgs() << "Allocator " << (void *)this << " added unfinalized eh-frame " 165 << formatv("[ {0:x} {1:x} ]", LoadAddr, LoadAddr + Size) << "\n"; 166 }); 167 std::lock_guard<std::mutex> Lock(M); 168 // Bail out early if there's already an error. 169 if (!ErrMsg.empty()) 170 return; 171 172 ExecutorAddr LA(LoadAddr); 173 for (auto &SecAllocGroup : llvm::reverse(Unfinalized)) { 174 if (SecAllocGroup.RemoteCode.contains(LA) || 175 SecAllocGroup.RemoteROData.contains(LA) || 176 SecAllocGroup.RemoteRWData.contains(LA)) { 177 SecAllocGroup.UnfinalizedEHFrames.push_back({LA, Size}); 178 return; 179 } 180 } 181 ErrMsg = "eh-frame does not lie inside unfinalized alloc"; 182 } 183 184 void EPCGenericRTDyldMemoryManager::deregisterEHFrames() { 185 // This is a no-op for us: We've registered a deallocation action for it. 186 } 187 188 void EPCGenericRTDyldMemoryManager::notifyObjectLoaded( 189 RuntimeDyld &Dyld, const object::ObjectFile &Obj) { 190 std::lock_guard<std::mutex> Lock(M); 191 LLVM_DEBUG(dbgs() << "Allocator " << (void *)this << " applied mappings:\n"); 192 for (auto &ObjAllocs : Unmapped) { 193 mapAllocsToRemoteAddrs(Dyld, ObjAllocs.CodeAllocs, 194 ObjAllocs.RemoteCode.Start); 195 mapAllocsToRemoteAddrs(Dyld, ObjAllocs.RODataAllocs, 196 ObjAllocs.RemoteROData.Start); 197 mapAllocsToRemoteAddrs(Dyld, ObjAllocs.RWDataAllocs, 198 ObjAllocs.RemoteRWData.Start); 199 Unfinalized.push_back(std::move(ObjAllocs)); 200 } 201 Unmapped.clear(); 202 } 203 204 bool EPCGenericRTDyldMemoryManager::finalizeMemory(std::string *ErrMsg) { 205 LLVM_DEBUG(dbgs() << "Allocator " << (void *)this << " finalizing:\n"); 206 207 // If there's an error then bail out here. 208 std::vector<SectionAllocGroup> SecAllocGroups; 209 { 210 std::lock_guard<std::mutex> Lock(M); 211 if (ErrMsg && !this->ErrMsg.empty()) { 212 *ErrMsg = std::move(this->ErrMsg); 213 return true; 214 } 215 std::swap(SecAllocGroups, Unfinalized); 216 } 217 218 // Loop over unfinalized objects to make finalization requests. 219 for (auto &SecAllocGroup : SecAllocGroups) { 220 221 MemProt SegMemProts[3] = {MemProt::Read | MemProt::Exec, MemProt::Read, 222 MemProt::Read | MemProt::Write}; 223 224 ExecutorAddrRange *RemoteAddrs[3] = {&SecAllocGroup.RemoteCode, 225 &SecAllocGroup.RemoteROData, 226 &SecAllocGroup.RemoteRWData}; 227 228 std::vector<SectionAlloc> *SegSections[3] = {&SecAllocGroup.CodeAllocs, 229 &SecAllocGroup.RODataAllocs, 230 &SecAllocGroup.RWDataAllocs}; 231 232 tpctypes::FinalizeRequest FR; 233 std::unique_ptr<char[]> AggregateContents[3]; 234 235 for (unsigned I = 0; I != 3; ++I) { 236 FR.Segments.push_back({}); 237 auto &Seg = FR.Segments.back(); 238 Seg.RAG = SegMemProts[I]; 239 Seg.Addr = RemoteAddrs[I]->Start; 240 for (auto &SecAlloc : *SegSections[I]) { 241 Seg.Size = alignTo(Seg.Size, SecAlloc.Align); 242 Seg.Size += SecAlloc.Size; 243 } 244 AggregateContents[I] = std::make_unique<char[]>(Seg.Size); 245 size_t SecOffset = 0; 246 for (auto &SecAlloc : *SegSections[I]) { 247 SecOffset = alignTo(SecOffset, SecAlloc.Align); 248 memcpy(&AggregateContents[I][SecOffset], 249 reinterpret_cast<const char *>( 250 alignAddr(SecAlloc.Contents.get(), Align(SecAlloc.Align))), 251 SecAlloc.Size); 252 SecOffset += SecAlloc.Size; 253 // FIXME: Can we reset SecAlloc.Content here, now that it's copied into 254 // the aggregated content? 255 } 256 Seg.Content = {AggregateContents[I].get(), SecOffset}; 257 } 258 259 for (auto &Frame : SecAllocGroup.UnfinalizedEHFrames) 260 FR.Actions.push_back( 261 {cantFail( 262 WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>( 263 SAs.RegisterEHFrame, Frame)), 264 cantFail( 265 WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>( 266 SAs.DeregisterEHFrame, Frame))}); 267 268 // We'll also need to make an extra allocation for the eh-frame wrapper call 269 // arguments. 270 Error FinalizeErr = Error::success(); 271 if (auto Err = EPC.callSPSWrapper< 272 rt::SPSSimpleExecutorMemoryManagerFinalizeSignature>( 273 SAs.Finalize, FinalizeErr, SAs.Instance, std::move(FR))) { 274 std::lock_guard<std::mutex> Lock(M); 275 this->ErrMsg = toString(std::move(Err)); 276 dbgs() << "Serialization error: " << this->ErrMsg << "\n"; 277 if (ErrMsg) 278 *ErrMsg = this->ErrMsg; 279 return true; 280 } 281 if (FinalizeErr) { 282 std::lock_guard<std::mutex> Lock(M); 283 this->ErrMsg = toString(std::move(FinalizeErr)); 284 dbgs() << "Finalization error: " << this->ErrMsg << "\n"; 285 if (ErrMsg) 286 *ErrMsg = this->ErrMsg; 287 return true; 288 } 289 } 290 291 return false; 292 } 293 294 void EPCGenericRTDyldMemoryManager::mapAllocsToRemoteAddrs( 295 RuntimeDyld &Dyld, std::vector<SectionAlloc> &Allocs, 296 ExecutorAddr NextAddr) { 297 for (auto &Alloc : Allocs) { 298 NextAddr.setValue(alignTo(NextAddr.getValue(), Alloc.Align)); 299 LLVM_DEBUG({ 300 dbgs() << " " << static_cast<void *>(Alloc.Contents.get()) << " -> " 301 << format("0x%016" PRIx64, NextAddr.getValue()) << "\n"; 302 }); 303 Dyld.mapSectionAddress(reinterpret_cast<const void *>(alignAddr( 304 Alloc.Contents.get(), Align(Alloc.Align))), 305 NextAddr.getValue()); 306 Alloc.RemoteAddr = NextAddr; 307 // Only advance NextAddr if it was non-null to begin with, 308 // otherwise leave it as null. 309 if (NextAddr) 310 NextAddr += ExecutorAddrDiff(Alloc.Size); 311 } 312 } 313 314 } // end namespace orc 315 } // end namespace llvm 316