1 //===----- EPCGenericRTDyldMemoryManager.cpp - EPC-bbasde MemMgr -----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h" 10 #include "llvm/ExecutionEngine/Orc/EPCGenericMemoryAccess.h" 11 #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" 12 #include "llvm/Support/Alignment.h" 13 #include "llvm/Support/FormatVariadic.h" 14 15 #define DEBUG_TYPE "orc" 16 17 using namespace llvm::orc::shared; 18 19 namespace llvm { 20 namespace orc { 21 22 Expected<std::unique_ptr<EPCGenericRTDyldMemoryManager>> 23 EPCGenericRTDyldMemoryManager::CreateWithDefaultBootstrapSymbols( 24 ExecutorProcessControl &EPC) { 25 SymbolAddrs SAs; 26 if (auto Err = EPC.getBootstrapSymbols( 27 {{SAs.Instance, rt::SimpleExecutorMemoryManagerInstanceName}, 28 {SAs.Reserve, rt::SimpleExecutorMemoryManagerReserveWrapperName}, 29 {SAs.Finalize, rt::SimpleExecutorMemoryManagerFinalizeWrapperName}, 30 {SAs.Deallocate, 31 rt::SimpleExecutorMemoryManagerDeallocateWrapperName}, 32 {SAs.RegisterEHFrame, rt::RegisterEHFrameSectionWrapperName}, 33 {SAs.DeregisterEHFrame, rt::DeregisterEHFrameSectionWrapperName}})) 34 return std::move(Err); 35 return std::make_unique<EPCGenericRTDyldMemoryManager>(EPC, std::move(SAs)); 36 } 37 38 EPCGenericRTDyldMemoryManager::EPCGenericRTDyldMemoryManager( 39 ExecutorProcessControl &EPC, SymbolAddrs SAs) 40 : EPC(EPC), SAs(std::move(SAs)) { 41 LLVM_DEBUG(dbgs() << "Created remote allocator " << (void *)this << "\n"); 42 } 43 44 EPCGenericRTDyldMemoryManager::~EPCGenericRTDyldMemoryManager() { 45 LLVM_DEBUG(dbgs() << "Destroyed remote allocator " << (void *)this << "\n"); 46 if (!ErrMsg.empty()) 47 errs() << "Destroying with existing errors:\n" << ErrMsg << "\n"; 48 49 Error Err = Error::success(); 50 if (auto Err2 = EPC.callSPSWrapper< 51 rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>( 52 SAs.Reserve, Err, SAs.Instance, FinalizedAllocs)) { 53 // FIXME: Report errors through EPC once that functionality is available. 54 logAllUnhandledErrors(std::move(Err2), errs(), ""); 55 return; 56 } 57 58 if (Err) 59 logAllUnhandledErrors(std::move(Err), errs(), ""); 60 } 61 62 uint8_t *EPCGenericRTDyldMemoryManager::allocateCodeSection( 63 uintptr_t Size, unsigned Alignment, unsigned SectionID, 64 StringRef SectionName) { 65 std::lock_guard<std::mutex> Lock(M); 66 LLVM_DEBUG({ 67 dbgs() << "Allocator " << (void *)this << " allocating code section " 68 << SectionName << ": size = " << formatv("{0:x}", Size) 69 << " bytes, alignment = " << Alignment << "\n"; 70 }); 71 auto &Seg = Unmapped.back().CodeAllocs; 72 Seg.emplace_back(Size, Alignment); 73 return reinterpret_cast<uint8_t *>( 74 alignAddr(Seg.back().Contents.get(), Align(Alignment))); 75 } 76 77 uint8_t *EPCGenericRTDyldMemoryManager::allocateDataSection( 78 uintptr_t Size, unsigned Alignment, unsigned SectionID, 79 StringRef SectionName, bool IsReadOnly) { 80 std::lock_guard<std::mutex> Lock(M); 81 LLVM_DEBUG({ 82 dbgs() << "Allocator " << (void *)this << " allocating " 83 << (IsReadOnly ? "ro" : "rw") << "-data section " << SectionName 84 << ": size = " << formatv("{0:x}", Size) << " bytes, alignment " 85 << Alignment << ")\n"; 86 }); 87 88 auto &Seg = 89 IsReadOnly ? Unmapped.back().RODataAllocs : Unmapped.back().RWDataAllocs; 90 91 Seg.emplace_back(Size, Alignment); 92 return reinterpret_cast<uint8_t *>( 93 alignAddr(Seg.back().Contents.get(), Align(Alignment))); 94 } 95 96 void EPCGenericRTDyldMemoryManager::reserveAllocationSpace( 97 uintptr_t CodeSize, uint32_t CodeAlign, uintptr_t RODataSize, 98 uint32_t RODataAlign, uintptr_t RWDataSize, uint32_t RWDataAlign) { 99 100 { 101 std::lock_guard<std::mutex> Lock(M); 102 // If there's already an error then bail out. 103 if (!ErrMsg.empty()) 104 return; 105 106 if (!isPowerOf2_32(CodeAlign) || CodeAlign > EPC.getPageSize()) { 107 ErrMsg = "Invalid code alignment in reserveAllocationSpace"; 108 return; 109 } 110 if (!isPowerOf2_32(RODataAlign) || RODataAlign > EPC.getPageSize()) { 111 ErrMsg = "Invalid ro-data alignment in reserveAllocationSpace"; 112 return; 113 } 114 if (!isPowerOf2_32(RWDataAlign) || RWDataAlign > EPC.getPageSize()) { 115 ErrMsg = "Invalid rw-data alignment in reserveAllocationSpace"; 116 return; 117 } 118 } 119 120 uint64_t TotalSize = 0; 121 TotalSize += alignTo(CodeSize, EPC.getPageSize()); 122 TotalSize += alignTo(RODataSize, EPC.getPageSize()); 123 TotalSize += alignTo(RWDataSize, EPC.getPageSize()); 124 125 LLVM_DEBUG({ 126 dbgs() << "Allocator " << (void *)this << " reserving " 127 << formatv("{0:x}", TotalSize) << " bytes.\n"; 128 }); 129 130 Expected<ExecutorAddr> TargetAllocAddr((ExecutorAddr())); 131 if (auto Err = EPC.callSPSWrapper< 132 rt::SPSSimpleExecutorMemoryManagerReserveSignature>( 133 SAs.Reserve, TargetAllocAddr, SAs.Instance, TotalSize)) { 134 std::lock_guard<std::mutex> Lock(M); 135 ErrMsg = toString(std::move(Err)); 136 return; 137 } 138 if (!TargetAllocAddr) { 139 std::lock_guard<std::mutex> Lock(M); 140 ErrMsg = toString(TargetAllocAddr.takeError()); 141 return; 142 } 143 144 std::lock_guard<std::mutex> Lock(M); 145 Unmapped.push_back(AllocGroup()); 146 Unmapped.back().RemoteCode = { 147 *TargetAllocAddr, ExecutorAddrDiff(alignTo(CodeSize, EPC.getPageSize()))}; 148 Unmapped.back().RemoteROData = { 149 Unmapped.back().RemoteCode.End, 150 ExecutorAddrDiff(alignTo(RODataSize, EPC.getPageSize()))}; 151 Unmapped.back().RemoteRWData = { 152 Unmapped.back().RemoteROData.End, 153 ExecutorAddrDiff(alignTo(RWDataSize, EPC.getPageSize()))}; 154 } 155 156 bool EPCGenericRTDyldMemoryManager::needsToReserveAllocationSpace() { 157 return true; 158 } 159 160 void EPCGenericRTDyldMemoryManager::registerEHFrames(uint8_t *Addr, 161 uint64_t LoadAddr, 162 size_t Size) { 163 LLVM_DEBUG({ 164 dbgs() << "Allocator " << (void *)this << " added unfinalized eh-frame " 165 << formatv("[ {0:x} {1:x} ]", LoadAddr, LoadAddr + Size) << "\n"; 166 }); 167 std::lock_guard<std::mutex> Lock(M); 168 // Bail out early if there's already an error. 169 if (!ErrMsg.empty()) 170 return; 171 172 ExecutorAddr LA(LoadAddr); 173 for (auto &Alloc : llvm::reverse(Unfinalized)) { 174 if (Alloc.RemoteCode.contains(LA) || Alloc.RemoteROData.contains(LA) || 175 Alloc.RemoteRWData.contains(LA)) { 176 Alloc.UnfinalizedEHFrames.push_back({LA, Size}); 177 return; 178 } 179 } 180 ErrMsg = "eh-frame does not lie inside unfinalized alloc"; 181 } 182 183 void EPCGenericRTDyldMemoryManager::deregisterEHFrames() { 184 // This is a no-op for us: We've registered a deallocation action for it. 185 } 186 187 void EPCGenericRTDyldMemoryManager::notifyObjectLoaded( 188 RuntimeDyld &Dyld, const object::ObjectFile &Obj) { 189 std::lock_guard<std::mutex> Lock(M); 190 LLVM_DEBUG(dbgs() << "Allocator " << (void *)this << " applied mappings:\n"); 191 for (auto &ObjAllocs : Unmapped) { 192 mapAllocsToRemoteAddrs(Dyld, ObjAllocs.CodeAllocs, 193 ObjAllocs.RemoteCode.Start); 194 mapAllocsToRemoteAddrs(Dyld, ObjAllocs.RODataAllocs, 195 ObjAllocs.RemoteROData.Start); 196 mapAllocsToRemoteAddrs(Dyld, ObjAllocs.RWDataAllocs, 197 ObjAllocs.RemoteRWData.Start); 198 Unfinalized.push_back(std::move(ObjAllocs)); 199 } 200 Unmapped.clear(); 201 } 202 203 bool EPCGenericRTDyldMemoryManager::finalizeMemory(std::string *ErrMsg) { 204 LLVM_DEBUG(dbgs() << "Allocator " << (void *)this << " finalizing:\n"); 205 206 // If there's an error then bail out here. 207 std::vector<AllocGroup> Allocs; 208 { 209 std::lock_guard<std::mutex> Lock(M); 210 if (ErrMsg && !this->ErrMsg.empty()) { 211 *ErrMsg = std::move(this->ErrMsg); 212 return true; 213 } 214 std::swap(Allocs, Unfinalized); 215 } 216 217 // Loop over unfinalized objects to make finalization requests. 218 for (auto &ObjAllocs : Allocs) { 219 220 tpctypes::WireProtectionFlags SegProts[3] = { 221 tpctypes::toWireProtectionFlags( 222 static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | 223 sys::Memory::MF_EXEC)), 224 tpctypes::toWireProtectionFlags(sys::Memory::MF_READ), 225 tpctypes::toWireProtectionFlags( 226 static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | 227 sys::Memory::MF_WRITE))}; 228 229 ExecutorAddrRange *RemoteAddrs[3] = {&ObjAllocs.RemoteCode, 230 &ObjAllocs.RemoteROData, 231 &ObjAllocs.RemoteRWData}; 232 233 std::vector<Alloc> *SegSections[3] = {&ObjAllocs.CodeAllocs, 234 &ObjAllocs.RODataAllocs, 235 &ObjAllocs.RWDataAllocs}; 236 237 tpctypes::FinalizeRequest FR; 238 std::unique_ptr<char[]> AggregateContents[3]; 239 240 for (unsigned I = 0; I != 3; ++I) { 241 FR.Segments.push_back({}); 242 auto &Seg = FR.Segments.back(); 243 Seg.Prot = SegProts[I]; 244 Seg.Addr = RemoteAddrs[I]->Start; 245 for (auto &SecAlloc : *SegSections[I]) { 246 Seg.Size = alignTo(Seg.Size, SecAlloc.Align); 247 Seg.Size += SecAlloc.Size; 248 } 249 AggregateContents[I] = std::make_unique<char[]>(Seg.Size); 250 size_t SecOffset = 0; 251 for (auto &SecAlloc : *SegSections[I]) { 252 SecOffset = alignTo(SecOffset, SecAlloc.Align); 253 memcpy(&AggregateContents[I][SecOffset], 254 reinterpret_cast<const char *>( 255 alignAddr(SecAlloc.Contents.get(), Align(SecAlloc.Align))), 256 SecAlloc.Size); 257 SecOffset += SecAlloc.Size; 258 // FIXME: Can we reset SecAlloc.Content here, now that it's copied into 259 // the aggregated content? 260 } 261 Seg.Content = {AggregateContents[I].get(), SecOffset}; 262 } 263 264 for (auto &Frame : ObjAllocs.UnfinalizedEHFrames) 265 FR.Actions.push_back( 266 {cantFail( 267 WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>( 268 SAs.RegisterEHFrame, Frame)), 269 cantFail( 270 WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>( 271 SAs.DeregisterEHFrame, Frame))}); 272 273 // We'll also need to make an extra allocation for the eh-frame wrapper call 274 // arguments. 275 Error FinalizeErr = Error::success(); 276 if (auto Err = EPC.callSPSWrapper< 277 rt::SPSSimpleExecutorMemoryManagerFinalizeSignature>( 278 SAs.Finalize, FinalizeErr, SAs.Instance, std::move(FR))) { 279 std::lock_guard<std::mutex> Lock(M); 280 this->ErrMsg = toString(std::move(Err)); 281 dbgs() << "Serialization error: " << this->ErrMsg << "\n"; 282 if (ErrMsg) 283 *ErrMsg = this->ErrMsg; 284 return true; 285 } 286 if (FinalizeErr) { 287 std::lock_guard<std::mutex> Lock(M); 288 this->ErrMsg = toString(std::move(FinalizeErr)); 289 dbgs() << "Finalization error: " << this->ErrMsg << "\n"; 290 if (ErrMsg) 291 *ErrMsg = this->ErrMsg; 292 return true; 293 } 294 } 295 296 return false; 297 } 298 299 void EPCGenericRTDyldMemoryManager::mapAllocsToRemoteAddrs( 300 RuntimeDyld &Dyld, std::vector<Alloc> &Allocs, ExecutorAddr NextAddr) { 301 for (auto &Alloc : Allocs) { 302 NextAddr.setValue(alignTo(NextAddr.getValue(), Alloc.Align)); 303 LLVM_DEBUG({ 304 dbgs() << " " << static_cast<void *>(Alloc.Contents.get()) << " -> " 305 << format("0x%016" PRIx64, NextAddr.getValue()) << "\n"; 306 }); 307 Dyld.mapSectionAddress(reinterpret_cast<const void *>(alignAddr( 308 Alloc.Contents.get(), Align(Alloc.Align))), 309 NextAddr.getValue()); 310 Alloc.RemoteAddr = NextAddr; 311 // Only advance NextAddr if it was non-null to begin with, 312 // otherwise leave it as null. 313 if (NextAddr) 314 NextAddr += ExecutorAddrDiff(Alloc.Size); 315 } 316 } 317 318 } // end namespace orc 319 } // end namespace llvm 320