1 //===- MemoryMapper.cpp - Cross-process memory mapper ------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/MemoryMapper.h" 10 11 #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" 12 #include "llvm/Support/WindowsError.h" 13 14 #include <algorithm> 15 16 #if defined(LLVM_ON_UNIX) && !defined(__ANDROID__) 17 #include <fcntl.h> 18 #include <sys/mman.h> 19 #include <unistd.h> 20 #elif defined(_WIN32) 21 #include <windows.h> 22 #endif 23 24 namespace llvm { 25 namespace orc { 26 27 MemoryMapper::~MemoryMapper() {} 28 29 InProcessMemoryMapper::InProcessMemoryMapper(size_t PageSize) 30 : PageSize(PageSize) {} 31 32 Expected<std::unique_ptr<InProcessMemoryMapper>> 33 InProcessMemoryMapper::Create() { 34 auto PageSize = sys::Process::getPageSize(); 35 if (!PageSize) 36 return PageSize.takeError(); 37 return std::make_unique<InProcessMemoryMapper>(*PageSize); 38 } 39 40 void InProcessMemoryMapper::reserve(size_t NumBytes, 41 OnReservedFunction OnReserved) { 42 std::error_code EC; 43 auto MB = sys::Memory::allocateMappedMemory( 44 NumBytes, nullptr, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC); 45 46 if (EC) 47 return OnReserved(errorCodeToError(EC)); 48 49 { 50 std::lock_guard<std::mutex> Lock(Mutex); 51 Reservations[MB.base()].Size = MB.allocatedSize(); 52 } 53 54 OnReserved( 55 ExecutorAddrRange(ExecutorAddr::fromPtr(MB.base()), MB.allocatedSize())); 56 } 57 58 char *InProcessMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) { 59 return Addr.toPtr<char *>(); 60 } 61 62 void InProcessMemoryMapper::initialize(MemoryMapper::AllocInfo &AI, 63 OnInitializedFunction OnInitialized) { 64 ExecutorAddr MinAddr(~0ULL); 65 ExecutorAddr MaxAddr(0); 66 67 // FIXME: Release finalize lifetime segments. 68 for (auto &Segment : AI.Segments) { 69 auto Base = AI.MappingBase + Segment.Offset; 70 auto Size = Segment.ContentSize + Segment.ZeroFillSize; 71 72 if (Base < MinAddr) 73 MinAddr = Base; 74 75 if (Base + Size > MaxAddr) 76 MaxAddr = Base + Size; 77 78 std::memset((Base + Segment.ContentSize).toPtr<void *>(), 0, 79 Segment.ZeroFillSize); 80 81 if (auto EC = sys::Memory::protectMappedMemory( 82 {Base.toPtr<void *>(), Size}, 83 toSysMemoryProtectionFlags(Segment.AG.getMemProt()))) { 84 return OnInitialized(errorCodeToError(EC)); 85 } 86 if ((Segment.AG.getMemProt() & MemProt::Exec) == MemProt::Exec) 87 sys::Memory::InvalidateInstructionCache(Base.toPtr<void *>(), Size); 88 } 89 90 auto DeinitializeActions = shared::runFinalizeActions(AI.Actions); 91 if (!DeinitializeActions) 92 return OnInitialized(DeinitializeActions.takeError()); 93 94 { 95 std::lock_guard<std::mutex> Lock(Mutex); 96 97 // This is the maximum range whose permission have been possibly modified 98 Allocations[MinAddr].Size = MaxAddr - MinAddr; 99 Allocations[MinAddr].DeinitializationActions = 100 std::move(*DeinitializeActions); 101 Reservations[AI.MappingBase.toPtr<void *>()].Allocations.push_back(MinAddr); 102 } 103 104 OnInitialized(MinAddr); 105 } 106 107 void InProcessMemoryMapper::deinitialize( 108 ArrayRef<ExecutorAddr> Bases, 109 MemoryMapper::OnDeinitializedFunction OnDeinitialized) { 110 Error AllErr = Error::success(); 111 112 { 113 std::lock_guard<std::mutex> Lock(Mutex); 114 115 for (auto Base : llvm::reverse(Bases)) { 116 117 if (Error Err = shared::runDeallocActions( 118 Allocations[Base].DeinitializationActions)) { 119 AllErr = joinErrors(std::move(AllErr), std::move(Err)); 120 } 121 122 // Reset protections to read/write so the area can be reused 123 if (auto EC = sys::Memory::protectMappedMemory( 124 {Base.toPtr<void *>(), Allocations[Base].Size}, 125 sys::Memory::ProtectionFlags::MF_READ | 126 sys::Memory::ProtectionFlags::MF_WRITE)) { 127 AllErr = joinErrors(std::move(AllErr), errorCodeToError(EC)); 128 } 129 130 Allocations.erase(Base); 131 } 132 } 133 134 OnDeinitialized(std::move(AllErr)); 135 } 136 137 void InProcessMemoryMapper::release(ArrayRef<ExecutorAddr> Bases, 138 OnReleasedFunction OnReleased) { 139 Error Err = Error::success(); 140 141 for (auto Base : Bases) { 142 std::vector<ExecutorAddr> AllocAddrs; 143 size_t Size; 144 { 145 std::lock_guard<std::mutex> Lock(Mutex); 146 auto &R = Reservations[Base.toPtr<void *>()]; 147 Size = R.Size; 148 AllocAddrs.swap(R.Allocations); 149 } 150 151 // deinitialize sub allocations 152 std::promise<MSVCPError> P; 153 auto F = P.get_future(); 154 deinitialize(AllocAddrs, [&](Error Err) { P.set_value(std::move(Err)); }); 155 if (Error E = F.get()) { 156 Err = joinErrors(std::move(Err), std::move(E)); 157 } 158 159 // free the memory 160 auto MB = sys::MemoryBlock(Base.toPtr<void *>(), Size); 161 162 auto EC = sys::Memory::releaseMappedMemory(MB); 163 if (EC) { 164 Err = joinErrors(std::move(Err), errorCodeToError(EC)); 165 } 166 167 std::lock_guard<std::mutex> Lock(Mutex); 168 Reservations.erase(Base.toPtr<void *>()); 169 } 170 171 OnReleased(std::move(Err)); 172 } 173 174 InProcessMemoryMapper::~InProcessMemoryMapper() { 175 std::vector<ExecutorAddr> ReservationAddrs; 176 { 177 std::lock_guard<std::mutex> Lock(Mutex); 178 179 ReservationAddrs.reserve(Reservations.size()); 180 for (const auto &R : Reservations) { 181 ReservationAddrs.push_back(ExecutorAddr::fromPtr(R.getFirst())); 182 } 183 } 184 185 std::promise<MSVCPError> P; 186 auto F = P.get_future(); 187 release(ReservationAddrs, [&](Error Err) { P.set_value(std::move(Err)); }); 188 cantFail(F.get()); 189 } 190 191 // SharedMemoryMapper 192 193 SharedMemoryMapper::SharedMemoryMapper(ExecutorProcessControl &EPC, 194 SymbolAddrs SAs, size_t PageSize) 195 : EPC(EPC), SAs(SAs), PageSize(PageSize) { 196 #if (!defined(LLVM_ON_UNIX) || defined(__ANDROID__)) && !defined(_WIN32) 197 llvm_unreachable("SharedMemoryMapper is not supported on this platform yet"); 198 #endif 199 } 200 201 Expected<std::unique_ptr<SharedMemoryMapper>> 202 SharedMemoryMapper::Create(ExecutorProcessControl &EPC, SymbolAddrs SAs) { 203 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32) 204 auto PageSize = sys::Process::getPageSize(); 205 if (!PageSize) 206 return PageSize.takeError(); 207 208 return std::make_unique<SharedMemoryMapper>(EPC, SAs, *PageSize); 209 #else 210 return make_error<StringError>( 211 "SharedMemoryMapper is not supported on this platform yet", 212 inconvertibleErrorCode()); 213 #endif 214 } 215 216 void SharedMemoryMapper::reserve(size_t NumBytes, 217 OnReservedFunction OnReserved) { 218 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32) 219 220 EPC.callSPSWrapperAsync< 221 rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>( 222 SAs.Reserve, 223 [this, NumBytes, OnReserved = std::move(OnReserved)]( 224 Error SerializationErr, 225 Expected<std::pair<ExecutorAddr, std::string>> Result) mutable { 226 if (SerializationErr) { 227 cantFail(Result.takeError()); 228 return OnReserved(std::move(SerializationErr)); 229 } 230 231 if (!Result) 232 return OnReserved(Result.takeError()); 233 234 ExecutorAddr RemoteAddr; 235 std::string SharedMemoryName; 236 std::tie(RemoteAddr, SharedMemoryName) = std::move(*Result); 237 238 void *LocalAddr = nullptr; 239 240 #if defined(LLVM_ON_UNIX) 241 242 int SharedMemoryFile = shm_open(SharedMemoryName.c_str(), O_RDWR, 0700); 243 if (SharedMemoryFile < 0) { 244 return OnReserved(errorCodeToError( 245 std::error_code(errno, std::generic_category()))); 246 } 247 248 // this prevents other processes from accessing it by name 249 shm_unlink(SharedMemoryName.c_str()); 250 251 LocalAddr = mmap(nullptr, NumBytes, PROT_READ | PROT_WRITE, MAP_SHARED, 252 SharedMemoryFile, 0); 253 if (LocalAddr == MAP_FAILED) { 254 return OnReserved(errorCodeToError( 255 std::error_code(errno, std::generic_category()))); 256 } 257 258 close(SharedMemoryFile); 259 260 #elif defined(_WIN32) 261 262 std::wstring WideSharedMemoryName(SharedMemoryName.begin(), 263 SharedMemoryName.end()); 264 HANDLE SharedMemoryFile = OpenFileMappingW( 265 FILE_MAP_ALL_ACCESS, FALSE, WideSharedMemoryName.c_str()); 266 if (!SharedMemoryFile) 267 return OnReserved(errorCodeToError(mapWindowsError(GetLastError()))); 268 269 LocalAddr = 270 MapViewOfFile(SharedMemoryFile, FILE_MAP_ALL_ACCESS, 0, 0, 0); 271 if (!LocalAddr) { 272 CloseHandle(SharedMemoryFile); 273 return OnReserved(errorCodeToError(mapWindowsError(GetLastError()))); 274 } 275 276 CloseHandle(SharedMemoryFile); 277 278 #endif 279 { 280 std::lock_guard<std::mutex> Lock(Mutex); 281 Reservations.insert({RemoteAddr, {LocalAddr, NumBytes}}); 282 } 283 284 OnReserved(ExecutorAddrRange(RemoteAddr, NumBytes)); 285 }, 286 SAs.Instance, static_cast<uint64_t>(NumBytes)); 287 288 #else 289 OnReserved(make_error<StringError>( 290 "SharedMemoryMapper is not supported on this platform yet", 291 inconvertibleErrorCode())); 292 #endif 293 } 294 295 char *SharedMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) { 296 auto R = Reservations.upper_bound(Addr); 297 assert(R != Reservations.begin() && "Attempt to prepare unreserved range"); 298 R--; 299 300 ExecutorAddrDiff Offset = Addr - R->first; 301 302 return static_cast<char *>(R->second.LocalAddr) + Offset; 303 } 304 305 void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI, 306 OnInitializedFunction OnInitialized) { 307 auto Reservation = Reservations.upper_bound(AI.MappingBase); 308 assert(Reservation != Reservations.begin() && "Attempt to initialize unreserved range"); 309 Reservation--; 310 311 auto AllocationOffset = AI.MappingBase - Reservation->first; 312 313 tpctypes::SharedMemoryFinalizeRequest FR; 314 315 AI.Actions.swap(FR.Actions); 316 317 FR.Segments.reserve(AI.Segments.size()); 318 319 for (auto Segment : AI.Segments) { 320 char *Base = static_cast<char *>(Reservation->second.LocalAddr) + 321 AllocationOffset + Segment.Offset; 322 std::memset(Base + Segment.ContentSize, 0, Segment.ZeroFillSize); 323 324 tpctypes::SharedMemorySegFinalizeRequest SegReq; 325 SegReq.RAG = {Segment.AG.getMemProt(), 326 Segment.AG.getMemLifetime() == MemLifetime::Finalize}; 327 SegReq.Addr = AI.MappingBase + Segment.Offset; 328 SegReq.Size = Segment.ContentSize + Segment.ZeroFillSize; 329 330 FR.Segments.push_back(SegReq); 331 } 332 333 EPC.callSPSWrapperAsync< 334 rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>( 335 SAs.Initialize, 336 [OnInitialized = std::move(OnInitialized)]( 337 Error SerializationErr, Expected<ExecutorAddr> Result) mutable { 338 if (SerializationErr) { 339 cantFail(Result.takeError()); 340 return OnInitialized(std::move(SerializationErr)); 341 } 342 343 OnInitialized(std::move(Result)); 344 }, 345 SAs.Instance, Reservation->first, std::move(FR)); 346 } 347 348 void SharedMemoryMapper::deinitialize( 349 ArrayRef<ExecutorAddr> Allocations, 350 MemoryMapper::OnDeinitializedFunction OnDeinitialized) { 351 EPC.callSPSWrapperAsync< 352 rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>( 353 SAs.Deinitialize, 354 [OnDeinitialized = std::move(OnDeinitialized)](Error SerializationErr, 355 Error Result) mutable { 356 if (SerializationErr) { 357 cantFail(std::move(Result)); 358 return OnDeinitialized(std::move(SerializationErr)); 359 } 360 361 OnDeinitialized(std::move(Result)); 362 }, 363 SAs.Instance, Allocations); 364 } 365 366 void SharedMemoryMapper::release(ArrayRef<ExecutorAddr> Bases, 367 OnReleasedFunction OnReleased) { 368 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32) 369 Error Err = Error::success(); 370 371 { 372 std::lock_guard<std::mutex> Lock(Mutex); 373 374 for (auto Base : Bases) { 375 376 #if defined(LLVM_ON_UNIX) 377 378 if (munmap(Reservations[Base].LocalAddr, Reservations[Base].Size) != 0) 379 Err = joinErrors(std::move(Err), errorCodeToError(std::error_code( 380 errno, std::generic_category()))); 381 382 #elif defined(_WIN32) 383 384 if (!UnmapViewOfFile(Reservations[Base].LocalAddr)) 385 Err = joinErrors(std::move(Err), 386 errorCodeToError(mapWindowsError(GetLastError()))); 387 388 #endif 389 390 Reservations.erase(Base); 391 } 392 } 393 394 EPC.callSPSWrapperAsync< 395 rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>( 396 SAs.Release, 397 [OnReleased = std::move(OnReleased), 398 Err = std::move(Err)](Error SerializationErr, Error Result) mutable { 399 if (SerializationErr) { 400 cantFail(std::move(Result)); 401 return OnReleased( 402 joinErrors(std::move(Err), std::move(SerializationErr))); 403 } 404 405 return OnReleased(joinErrors(std::move(Err), std::move(Result))); 406 }, 407 SAs.Instance, Bases); 408 #else 409 OnReleased(make_error<StringError>( 410 "SharedMemoryMapper is not supported on this platform yet", 411 inconvertibleErrorCode())); 412 #endif 413 } 414 415 SharedMemoryMapper::~SharedMemoryMapper() { 416 std::lock_guard<std::mutex> Lock(Mutex); 417 for (const auto &R : Reservations) { 418 419 #if defined(LLVM_ON_UNIX) && !defined(__ANDROID__) 420 421 munmap(R.second.LocalAddr, R.second.Size); 422 423 #elif defined(_WIN32) 424 425 UnmapViewOfFile(R.second.LocalAddr); 426 427 #else 428 429 (void)R; 430 431 #endif 432 } 433 } 434 435 } // namespace orc 436 437 } // namespace llvm 438