1 //===- SectionMemoryManager.cpp - Memory manager for MCJIT/RtDyld *- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the section-based memory manager used by the MCJIT 10 // execution engine and RuntimeDyld 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/ExecutionEngine/SectionMemoryManager.h" 15 #include "llvm/Config/config.h" 16 #include "llvm/Support/MathExtras.h" 17 #include "llvm/Support/Process.h" 18 19 namespace llvm { 20 21 uint8_t *SectionMemoryManager::allocateDataSection(uintptr_t Size, 22 unsigned Alignment, 23 unsigned SectionID, 24 StringRef SectionName, 25 bool IsReadOnly) { 26 if (IsReadOnly) 27 return allocateSection(SectionMemoryManager::AllocationPurpose::ROData, 28 Size, Alignment); 29 return allocateSection(SectionMemoryManager::AllocationPurpose::RWData, Size, 30 Alignment); 31 } 32 33 uint8_t *SectionMemoryManager::allocateCodeSection(uintptr_t Size, 34 unsigned Alignment, 35 unsigned SectionID, 36 StringRef SectionName) { 37 return allocateSection(SectionMemoryManager::AllocationPurpose::Code, Size, 38 Alignment); 39 } 40 41 uint8_t *SectionMemoryManager::allocateSection( 42 SectionMemoryManager::AllocationPurpose Purpose, uintptr_t Size, 43 unsigned Alignment) { 44 if (!Alignment) 45 Alignment = 16; 46 47 assert(!(Alignment & (Alignment - 1)) && "Alignment must be a power of two."); 48 49 uintptr_t RequiredSize = Alignment * ((Size + Alignment - 1) / Alignment + 1); 50 uintptr_t Addr = 0; 51 52 MemoryGroup &MemGroup = [&]() -> MemoryGroup & { 53 switch (Purpose) { 54 case AllocationPurpose::Code: 55 return CodeMem; 56 case AllocationPurpose::ROData: 57 return RODataMem; 58 case AllocationPurpose::RWData: 59 return RWDataMem; 60 } 61 llvm_unreachable("Unknown SectionMemoryManager::AllocationPurpose"); 62 }(); 63 64 // Look in the list of free memory regions and use a block there if one 65 // is available. 66 for (FreeMemBlock &FreeMB : MemGroup.FreeMem) { 67 if (FreeMB.Free.allocatedSize() >= RequiredSize) { 68 Addr = (uintptr_t)FreeMB.Free.base(); 69 uintptr_t EndOfBlock = Addr + FreeMB.Free.allocatedSize(); 70 // Align the address. 71 Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); 72 73 if (FreeMB.PendingPrefixIndex == (unsigned)-1) { 74 // The part of the block we're giving out to the user is now pending 75 MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size)); 76 77 // Remember this pending block, such that future allocations can just 78 // modify it rather than creating a new one 79 FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1; 80 } else { 81 sys::MemoryBlock &PendingMB = 82 MemGroup.PendingMem[FreeMB.PendingPrefixIndex]; 83 PendingMB = sys::MemoryBlock(PendingMB.base(), 84 Addr + Size - (uintptr_t)PendingMB.base()); 85 } 86 87 // Remember how much free space is now left in this block 88 FreeMB.Free = 89 sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size); 90 return (uint8_t *)Addr; 91 } 92 } 93 94 // No pre-allocated free block was large enough. Allocate a new memory region. 95 // Note that all sections get allocated as read-write. The permissions will 96 // be updated later based on memory group. 97 // 98 // FIXME: It would be useful to define a default allocation size (or add 99 // it as a constructor parameter) to minimize the number of allocations. 100 // 101 // FIXME: Initialize the Near member for each memory group to avoid 102 // interleaving. 103 std::error_code ec; 104 sys::MemoryBlock MB = MMapper->allocateMappedMemory( 105 Purpose, RequiredSize, &MemGroup.Near, 106 sys::Memory::MF_READ | sys::Memory::MF_WRITE, ec); 107 if (ec) { 108 // FIXME: Add error propagation to the interface. 109 return nullptr; 110 } 111 112 // Save this address as the basis for our next request 113 MemGroup.Near = MB; 114 115 // Copy the address to all the other groups, if they have not 116 // been initialized. 117 if (CodeMem.Near.base() == nullptr) 118 CodeMem.Near = MB; 119 if (RODataMem.Near.base() == nullptr) 120 RODataMem.Near = MB; 121 if (RWDataMem.Near.base() == nullptr) 122 RWDataMem.Near = MB; 123 124 // Remember that we allocated this memory 125 MemGroup.AllocatedMem.push_back(MB); 126 Addr = (uintptr_t)MB.base(); 127 uintptr_t EndOfBlock = Addr + MB.allocatedSize(); 128 129 // Align the address. 130 Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); 131 132 // The part of the block we're giving out to the user is now pending 133 MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size)); 134 135 // The allocateMappedMemory may allocate much more memory than we need. In 136 // this case, we store the unused memory as a free memory block. 137 unsigned FreeSize = EndOfBlock - Addr - Size; 138 if (FreeSize > 16) { 139 FreeMemBlock FreeMB; 140 FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), FreeSize); 141 FreeMB.PendingPrefixIndex = (unsigned)-1; 142 MemGroup.FreeMem.push_back(FreeMB); 143 } 144 145 // Return aligned address 146 return (uint8_t *)Addr; 147 } 148 149 bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) { 150 // FIXME: Should in-progress permissions be reverted if an error occurs? 151 std::error_code ec; 152 153 // Make code memory executable. 154 ec = applyMemoryGroupPermissions(CodeMem, 155 sys::Memory::MF_READ | sys::Memory::MF_EXEC); 156 if (ec) { 157 if (ErrMsg) { 158 *ErrMsg = ec.message(); 159 } 160 return true; 161 } 162 163 // Make read-only data memory read-only. 164 ec = applyMemoryGroupPermissions(RODataMem, sys::Memory::MF_READ); 165 if (ec) { 166 if (ErrMsg) { 167 *ErrMsg = ec.message(); 168 } 169 return true; 170 } 171 172 // Read-write data memory already has the correct permissions 173 174 // Some platforms with separate data cache and instruction cache require 175 // explicit cache flush, otherwise JIT code manipulations (like resolved 176 // relocations) will get to the data cache but not to the instruction cache. 177 invalidateInstructionCache(); 178 179 return false; 180 } 181 182 static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) { 183 static const size_t PageSize = sys::Process::getPageSizeEstimate(); 184 185 size_t StartOverlap = 186 (PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize; 187 188 size_t TrimmedSize = M.allocatedSize(); 189 TrimmedSize -= StartOverlap; 190 TrimmedSize -= TrimmedSize % PageSize; 191 192 sys::MemoryBlock Trimmed((void *)((uintptr_t)M.base() + StartOverlap), 193 TrimmedSize); 194 195 assert(((uintptr_t)Trimmed.base() % PageSize) == 0); 196 assert((Trimmed.allocatedSize() % PageSize) == 0); 197 assert(M.base() <= Trimmed.base() && 198 Trimmed.allocatedSize() <= M.allocatedSize()); 199 200 return Trimmed; 201 } 202 203 std::error_code 204 SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup, 205 unsigned Permissions) { 206 for (sys::MemoryBlock &MB : MemGroup.PendingMem) 207 if (std::error_code EC = MMapper->protectMappedMemory(MB, Permissions)) 208 return EC; 209 210 MemGroup.PendingMem.clear(); 211 212 // Now go through free blocks and trim any of them that don't span the entire 213 // page because one of the pending blocks may have overlapped it. 214 for (FreeMemBlock &FreeMB : MemGroup.FreeMem) { 215 FreeMB.Free = trimBlockToPageSize(FreeMB.Free); 216 // We cleared the PendingMem list, so all these pointers are now invalid 217 FreeMB.PendingPrefixIndex = (unsigned)-1; 218 } 219 220 // Remove all blocks which are now empty 221 erase_if(MemGroup.FreeMem, [](FreeMemBlock &FreeMB) { 222 return FreeMB.Free.allocatedSize() == 0; 223 }); 224 225 return std::error_code(); 226 } 227 228 void SectionMemoryManager::invalidateInstructionCache() { 229 for (sys::MemoryBlock &Block : CodeMem.PendingMem) 230 sys::Memory::InvalidateInstructionCache(Block.base(), 231 Block.allocatedSize()); 232 } 233 234 SectionMemoryManager::~SectionMemoryManager() { 235 for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) { 236 for (sys::MemoryBlock &Block : Group->AllocatedMem) 237 MMapper->releaseMappedMemory(Block); 238 } 239 } 240 241 SectionMemoryManager::MemoryMapper::~MemoryMapper() = default; 242 243 void SectionMemoryManager::anchor() {} 244 245 namespace { 246 // Trivial implementation of SectionMemoryManager::MemoryMapper that just calls 247 // into sys::Memory. 248 class DefaultMMapper final : public SectionMemoryManager::MemoryMapper { 249 public: 250 sys::MemoryBlock 251 allocateMappedMemory(SectionMemoryManager::AllocationPurpose Purpose, 252 size_t NumBytes, const sys::MemoryBlock *const NearBlock, 253 unsigned Flags, std::error_code &EC) override { 254 return sys::Memory::allocateMappedMemory(NumBytes, NearBlock, Flags, EC); 255 } 256 257 std::error_code protectMappedMemory(const sys::MemoryBlock &Block, 258 unsigned Flags) override { 259 return sys::Memory::protectMappedMemory(Block, Flags); 260 } 261 262 std::error_code releaseMappedMemory(sys::MemoryBlock &M) override { 263 return sys::Memory::releaseMappedMemory(M); 264 } 265 }; 266 } // namespace 267 268 SectionMemoryManager::SectionMemoryManager(MemoryMapper *UnownedMM) 269 : MMapper(UnownedMM), OwnedMMapper(nullptr) { 270 if (!MMapper) { 271 OwnedMMapper = std::make_unique<DefaultMMapper>(); 272 MMapper = OwnedMMapper.get(); 273 } 274 } 275 276 } // namespace llvm 277