1 //===- MemoryMapper.cpp - Cross-process memory mapper ------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/MemoryMapper.h" 10 11 #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" 12 #include "llvm/Support/WindowsError.h" 13 14 #include <algorithm> 15 16 #if defined(LLVM_ON_UNIX) && !defined(__ANDROID__) 17 #include <fcntl.h> 18 #include <sys/mman.h> 19 #include <unistd.h> 20 #elif defined(_WIN32) 21 #include <windows.h> 22 #endif 23 24 namespace llvm { 25 namespace orc { 26 27 MemoryMapper::~MemoryMapper() {} 28 29 InProcessMemoryMapper::InProcessMemoryMapper(size_t PageSize) 30 : PageSize(PageSize) {} 31 32 Expected<std::unique_ptr<InProcessMemoryMapper>> 33 InProcessMemoryMapper::Create() { 34 auto PageSize = sys::Process::getPageSize(); 35 if (!PageSize) 36 return PageSize.takeError(); 37 return std::make_unique<InProcessMemoryMapper>(*PageSize); 38 } 39 40 void InProcessMemoryMapper::reserve(size_t NumBytes, 41 OnReservedFunction OnReserved) { 42 std::error_code EC; 43 auto MB = sys::Memory::allocateMappedMemory( 44 NumBytes, nullptr, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC); 45 46 if (EC) 47 return OnReserved(errorCodeToError(EC)); 48 49 { 50 std::lock_guard<std::mutex> Lock(Mutex); 51 Reservations[MB.base()].Size = MB.allocatedSize(); 52 } 53 54 OnReserved( 55 ExecutorAddrRange(ExecutorAddr::fromPtr(MB.base()), MB.allocatedSize())); 56 } 57 58 char *InProcessMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) { 59 return Addr.toPtr<char *>(); 60 } 61 62 void InProcessMemoryMapper::initialize(MemoryMapper::AllocInfo &AI, 63 OnInitializedFunction OnInitialized) { 64 ExecutorAddr MinAddr(~0ULL); 65 ExecutorAddr MaxAddr(0); 66 67 // FIXME: Release finalize lifetime segments. 68 for (auto &Segment : AI.Segments) { 69 auto Base = AI.MappingBase + Segment.Offset; 70 auto Size = Segment.ContentSize + Segment.ZeroFillSize; 71 72 if (Base < MinAddr) 73 MinAddr = Base; 74 75 if (Base + Size > MaxAddr) 76 MaxAddr = Base + Size; 77 78 std::memset((Base + Segment.ContentSize).toPtr<void *>(), 0, 79 Segment.ZeroFillSize); 80 81 if (auto EC = sys::Memory::protectMappedMemory( 82 {Base.toPtr<void *>(), Size}, 83 toSysMemoryProtectionFlags(Segment.AG.getMemProt()))) { 84 return OnInitialized(errorCodeToError(EC)); 85 } 86 if ((Segment.AG.getMemProt() & MemProt::Exec) == MemProt::Exec) 87 sys::Memory::InvalidateInstructionCache(Base.toPtr<void *>(), Size); 88 } 89 90 auto DeinitializeActions = shared::runFinalizeActions(AI.Actions); 91 if (!DeinitializeActions) 92 return OnInitialized(DeinitializeActions.takeError()); 93 94 { 95 std::lock_guard<std::mutex> Lock(Mutex); 96 97 // This is the maximum range whose permission have been possibly modified 98 Allocations[MinAddr].Size = MaxAddr - MinAddr; 99 Allocations[MinAddr].DeinitializationActions = 100 std::move(*DeinitializeActions); 101 Reservations[AI.MappingBase.toPtr<void *>()].Allocations.push_back(MinAddr); 102 } 103 104 OnInitialized(MinAddr); 105 } 106 107 void InProcessMemoryMapper::deinitialize( 108 ArrayRef<ExecutorAddr> Bases, 109 MemoryMapper::OnDeinitializedFunction OnDeinitialized) { 110 Error AllErr = Error::success(); 111 112 { 113 std::lock_guard<std::mutex> Lock(Mutex); 114 115 for (auto Base : llvm::reverse(Bases)) { 116 117 if (Error Err = shared::runDeallocActions( 118 Allocations[Base].DeinitializationActions)) { 119 AllErr = joinErrors(std::move(AllErr), std::move(Err)); 120 } 121 122 // Reset protections to read/write so the area can be reused 123 if (auto EC = sys::Memory::protectMappedMemory( 124 {Base.toPtr<void *>(), Allocations[Base].Size}, 125 sys::Memory::ProtectionFlags::MF_READ | 126 sys::Memory::ProtectionFlags::MF_WRITE)) { 127 AllErr = joinErrors(std::move(AllErr), errorCodeToError(EC)); 128 } 129 130 Allocations.erase(Base); 131 } 132 } 133 134 OnDeinitialized(std::move(AllErr)); 135 } 136 137 void InProcessMemoryMapper::release(ArrayRef<ExecutorAddr> Bases, 138 OnReleasedFunction OnReleased) { 139 Error Err = Error::success(); 140 141 for (auto Base : Bases) { 142 std::vector<ExecutorAddr> AllocAddrs; 143 size_t Size; 144 { 145 std::lock_guard<std::mutex> Lock(Mutex); 146 auto &R = Reservations[Base.toPtr<void *>()]; 147 Size = R.Size; 148 AllocAddrs.swap(R.Allocations); 149 } 150 151 // deinitialize sub allocations 152 std::promise<MSVCPError> P; 153 auto F = P.get_future(); 154 deinitialize(AllocAddrs, [&](Error Err) { P.set_value(std::move(Err)); }); 155 if (Error E = F.get()) { 156 Err = joinErrors(std::move(Err), std::move(E)); 157 } 158 159 // free the memory 160 auto MB = sys::MemoryBlock(Base.toPtr<void *>(), Size); 161 162 auto EC = sys::Memory::releaseMappedMemory(MB); 163 if (EC) { 164 Err = joinErrors(std::move(Err), errorCodeToError(EC)); 165 } 166 167 std::lock_guard<std::mutex> Lock(Mutex); 168 Reservations.erase(Base.toPtr<void *>()); 169 } 170 171 OnReleased(std::move(Err)); 172 } 173 174 InProcessMemoryMapper::~InProcessMemoryMapper() { 175 std::vector<ExecutorAddr> ReservationAddrs; 176 { 177 std::lock_guard<std::mutex> Lock(Mutex); 178 179 ReservationAddrs.reserve(Reservations.size()); 180 for (const auto &R : Reservations) { 181 ReservationAddrs.push_back(ExecutorAddr::fromPtr(R.getFirst())); 182 } 183 } 184 185 std::promise<MSVCPError> P; 186 auto F = P.get_future(); 187 release(ReservationAddrs, [&](Error Err) { P.set_value(std::move(Err)); }); 188 cantFail(F.get()); 189 } 190 191 // SharedMemoryMapper 192 193 SharedMemoryMapper::SharedMemoryMapper(ExecutorProcessControl &EPC, 194 SymbolAddrs SAs, size_t PageSize) 195 : EPC(EPC), SAs(SAs), PageSize(PageSize) { 196 #if (!defined(LLVM_ON_UNIX) || defined(__ANDROID__)) && !defined(_WIN32) 197 llvm_unreachable("SharedMemoryMapper is not supported on this platform yet"); 198 #endif 199 } 200 201 Expected<std::unique_ptr<SharedMemoryMapper>> 202 SharedMemoryMapper::Create(ExecutorProcessControl &EPC, SymbolAddrs SAs) { 203 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32) 204 auto PageSize = sys::Process::getPageSize(); 205 if (!PageSize) 206 return PageSize.takeError(); 207 208 return std::make_unique<SharedMemoryMapper>(EPC, SAs, *PageSize); 209 #else 210 return make_error<StringError>( 211 "SharedMemoryMapper is not supported on this platform yet", 212 inconvertibleErrorCode()); 213 #endif 214 } 215 216 void SharedMemoryMapper::reserve(size_t NumBytes, 217 OnReservedFunction OnReserved) { 218 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32) 219 220 EPC.callSPSWrapperAsync< 221 rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>( 222 SAs.Reserve, 223 [this, NumBytes, OnReserved = std::move(OnReserved)]( 224 Error SerializationErr, 225 Expected<std::pair<ExecutorAddr, std::string>> Result) mutable { 226 if (SerializationErr) { 227 cantFail(Result.takeError()); 228 return OnReserved(std::move(SerializationErr)); 229 } 230 231 if (!Result) 232 return OnReserved(Result.takeError()); 233 234 ExecutorAddr RemoteAddr; 235 std::string SharedMemoryName; 236 std::tie(RemoteAddr, SharedMemoryName) = std::move(*Result); 237 238 void *LocalAddr = nullptr; 239 240 #if defined(LLVM_ON_UNIX) 241 242 int SharedMemoryFile = shm_open(SharedMemoryName.c_str(), O_RDWR, 0700); 243 if (SharedMemoryFile < 0) { 244 return OnReserved(errorCodeToError( 245 std::error_code(errno, std::generic_category()))); 246 } 247 248 // this prevents other processes from accessing it by name 249 shm_unlink(SharedMemoryName.c_str()); 250 251 LocalAddr = mmap(nullptr, NumBytes, PROT_READ | PROT_WRITE, MAP_SHARED, 252 SharedMemoryFile, 0); 253 if (LocalAddr == MAP_FAILED) { 254 return OnReserved(errorCodeToError( 255 std::error_code(errno, std::generic_category()))); 256 } 257 258 close(SharedMemoryFile); 259 260 #elif defined(_WIN32) 261 262 std::wstring WideSharedMemoryName(SharedMemoryName.begin(), 263 SharedMemoryName.end()); 264 HANDLE SharedMemoryFile = OpenFileMappingW( 265 FILE_MAP_ALL_ACCESS, FALSE, WideSharedMemoryName.c_str()); 266 if (!SharedMemoryFile) 267 return OnReserved(errorCodeToError(mapWindowsError(GetLastError()))); 268 269 LocalAddr = 270 MapViewOfFile(SharedMemoryFile, FILE_MAP_ALL_ACCESS, 0, 0, 0); 271 if (!LocalAddr) { 272 CloseHandle(SharedMemoryFile); 273 return OnReserved(errorCodeToError(mapWindowsError(GetLastError()))); 274 } 275 276 CloseHandle(SharedMemoryFile); 277 278 #endif 279 { 280 std::lock_guard<std::mutex> Lock(Mutex); 281 Reservations.insert({RemoteAddr, {LocalAddr, NumBytes}}); 282 } 283 284 OnReserved(ExecutorAddrRange(RemoteAddr, NumBytes)); 285 }, 286 SAs.Instance, static_cast<uint64_t>(NumBytes)); 287 288 #else 289 OnReserved(make_error<StringError>( 290 "SharedMemoryMapper is not supported on this platform yet", 291 inconvertibleErrorCode())); 292 #endif 293 } 294 295 char *SharedMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) { 296 auto R = Reservations.upper_bound(Addr); 297 assert(R != Reservations.begin() && "Attempt to prepare unreserved range"); 298 R--; 299 300 ExecutorAddrDiff Offset = Addr - R->first; 301 302 return static_cast<char *>(R->second.LocalAddr) + Offset; 303 } 304 305 void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI, 306 OnInitializedFunction OnInitialized) { 307 auto Reservation = Reservations.upper_bound(AI.MappingBase); 308 assert(Reservation != Reservations.begin() && "Attempt to initialize unreserved range"); 309 Reservation--; 310 311 auto AllocationOffset = AI.MappingBase - Reservation->first; 312 313 tpctypes::SharedMemoryFinalizeRequest FR; 314 315 AI.Actions.swap(FR.Actions); 316 317 FR.Segments.reserve(AI.Segments.size()); 318 319 for (auto Segment : AI.Segments) { 320 char *Base = static_cast<char *>(Reservation->second.LocalAddr) + 321 AllocationOffset + Segment.Offset; 322 std::memset(Base + Segment.ContentSize, 0, Segment.ZeroFillSize); 323 324 tpctypes::SharedMemorySegFinalizeRequest SegReq; 325 SegReq.AG = Segment.AG; 326 SegReq.Addr = AI.MappingBase + Segment.Offset; 327 SegReq.Size = Segment.ContentSize + Segment.ZeroFillSize; 328 329 FR.Segments.push_back(SegReq); 330 } 331 332 EPC.callSPSWrapperAsync< 333 rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>( 334 SAs.Initialize, 335 [OnInitialized = std::move(OnInitialized)]( 336 Error SerializationErr, Expected<ExecutorAddr> Result) mutable { 337 if (SerializationErr) { 338 cantFail(Result.takeError()); 339 return OnInitialized(std::move(SerializationErr)); 340 } 341 342 OnInitialized(std::move(Result)); 343 }, 344 SAs.Instance, Reservation->first, std::move(FR)); 345 } 346 347 void SharedMemoryMapper::deinitialize( 348 ArrayRef<ExecutorAddr> Allocations, 349 MemoryMapper::OnDeinitializedFunction OnDeinitialized) { 350 EPC.callSPSWrapperAsync< 351 rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>( 352 SAs.Deinitialize, 353 [OnDeinitialized = std::move(OnDeinitialized)](Error SerializationErr, 354 Error Result) mutable { 355 if (SerializationErr) { 356 cantFail(std::move(Result)); 357 return OnDeinitialized(std::move(SerializationErr)); 358 } 359 360 OnDeinitialized(std::move(Result)); 361 }, 362 SAs.Instance, Allocations); 363 } 364 365 void SharedMemoryMapper::release(ArrayRef<ExecutorAddr> Bases, 366 OnReleasedFunction OnReleased) { 367 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32) 368 Error Err = Error::success(); 369 370 { 371 std::lock_guard<std::mutex> Lock(Mutex); 372 373 for (auto Base : Bases) { 374 375 #if defined(LLVM_ON_UNIX) 376 377 if (munmap(Reservations[Base].LocalAddr, Reservations[Base].Size) != 0) 378 Err = joinErrors(std::move(Err), errorCodeToError(std::error_code( 379 errno, std::generic_category()))); 380 381 #elif defined(_WIN32) 382 383 if (!UnmapViewOfFile(Reservations[Base].LocalAddr)) 384 Err = joinErrors(std::move(Err), 385 errorCodeToError(mapWindowsError(GetLastError()))); 386 387 #endif 388 389 Reservations.erase(Base); 390 } 391 } 392 393 EPC.callSPSWrapperAsync< 394 rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>( 395 SAs.Release, 396 [OnReleased = std::move(OnReleased), 397 Err = std::move(Err)](Error SerializationErr, Error Result) mutable { 398 if (SerializationErr) { 399 cantFail(std::move(Result)); 400 return OnReleased( 401 joinErrors(std::move(Err), std::move(SerializationErr))); 402 } 403 404 return OnReleased(joinErrors(std::move(Err), std::move(Result))); 405 }, 406 SAs.Instance, Bases); 407 #else 408 OnReleased(make_error<StringError>( 409 "SharedMemoryMapper is not supported on this platform yet", 410 inconvertibleErrorCode())); 411 #endif 412 } 413 414 SharedMemoryMapper::~SharedMemoryMapper() { 415 std::lock_guard<std::mutex> Lock(Mutex); 416 for (const auto &R : Reservations) { 417 418 #if defined(LLVM_ON_UNIX) && !defined(__ANDROID__) 419 420 munmap(R.second.LocalAddr, R.second.Size); 421 422 #elif defined(_WIN32) 423 424 UnmapViewOfFile(R.second.LocalAddr); 425 426 #else 427 428 (void)R; 429 430 #endif 431 } 432 } 433 434 } // namespace orc 435 436 } // namespace llvm 437