1 //===- MemoryMapper.cpp - Cross-process memory mapper ------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/MemoryMapper.h" 10 11 #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" 12 #include "llvm/Support/WindowsError.h" 13 14 #include <algorithm> 15 16 #if defined(LLVM_ON_UNIX) && !defined(__ANDROID__) 17 #include <fcntl.h> 18 #include <sys/mman.h> 19 #if defined(__MVS__) 20 #include "llvm/Support/BLAKE3.h" 21 #include <sys/shm.h> 22 #endif 23 #include <unistd.h> 24 #elif defined(_WIN32) 25 #include <windows.h> 26 #endif 27 28 namespace llvm { 29 namespace orc { 30 31 MemoryMapper::~MemoryMapper() {} 32 33 InProcessMemoryMapper::InProcessMemoryMapper(size_t PageSize) 34 : PageSize(PageSize) {} 35 36 Expected<std::unique_ptr<InProcessMemoryMapper>> 37 InProcessMemoryMapper::Create() { 38 auto PageSize = sys::Process::getPageSize(); 39 if (!PageSize) 40 return PageSize.takeError(); 41 return std::make_unique<InProcessMemoryMapper>(*PageSize); 42 } 43 44 void InProcessMemoryMapper::reserve(size_t NumBytes, 45 OnReservedFunction OnReserved) { 46 std::error_code EC; 47 auto MB = sys::Memory::allocateMappedMemory( 48 NumBytes, nullptr, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC); 49 50 if (EC) 51 return OnReserved(errorCodeToError(EC)); 52 53 { 54 std::lock_guard<std::mutex> Lock(Mutex); 55 Reservations[MB.base()].Size = MB.allocatedSize(); 56 } 57 58 OnReserved( 59 ExecutorAddrRange(ExecutorAddr::fromPtr(MB.base()), MB.allocatedSize())); 60 } 61 62 char *InProcessMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) { 63 return Addr.toPtr<char *>(); 64 } 65 66 void InProcessMemoryMapper::initialize(MemoryMapper::AllocInfo &AI, 67 OnInitializedFunction OnInitialized) { 68 ExecutorAddr MinAddr(~0ULL); 69 ExecutorAddr MaxAddr(0); 70 71 // FIXME: Release finalize lifetime segments. 72 for (auto &Segment : AI.Segments) { 73 auto Base = AI.MappingBase + Segment.Offset; 74 auto Size = Segment.ContentSize + Segment.ZeroFillSize; 75 76 if (Base < MinAddr) 77 MinAddr = Base; 78 79 if (Base + Size > MaxAddr) 80 MaxAddr = Base + Size; 81 82 std::memset((Base + Segment.ContentSize).toPtr<void *>(), 0, 83 Segment.ZeroFillSize); 84 85 if (auto EC = sys::Memory::protectMappedMemory( 86 {Base.toPtr<void *>(), Size}, 87 toSysMemoryProtectionFlags(Segment.AG.getMemProt()))) { 88 return OnInitialized(errorCodeToError(EC)); 89 } 90 if ((Segment.AG.getMemProt() & MemProt::Exec) == MemProt::Exec) 91 sys::Memory::InvalidateInstructionCache(Base.toPtr<void *>(), Size); 92 } 93 94 auto DeinitializeActions = shared::runFinalizeActions(AI.Actions); 95 if (!DeinitializeActions) 96 return OnInitialized(DeinitializeActions.takeError()); 97 98 { 99 std::lock_guard<std::mutex> Lock(Mutex); 100 101 // This is the maximum range whose permission have been possibly modified 102 Allocations[MinAddr].Size = MaxAddr - MinAddr; 103 Allocations[MinAddr].DeinitializationActions = 104 std::move(*DeinitializeActions); 105 Reservations[AI.MappingBase.toPtr<void *>()].Allocations.push_back(MinAddr); 106 } 107 108 OnInitialized(MinAddr); 109 } 110 111 void InProcessMemoryMapper::deinitialize( 112 ArrayRef<ExecutorAddr> Bases, 113 MemoryMapper::OnDeinitializedFunction OnDeinitialized) { 114 Error AllErr = Error::success(); 115 116 { 117 std::lock_guard<std::mutex> Lock(Mutex); 118 119 for (auto Base : llvm::reverse(Bases)) { 120 121 if (Error Err = shared::runDeallocActions( 122 Allocations[Base].DeinitializationActions)) { 123 AllErr = joinErrors(std::move(AllErr), std::move(Err)); 124 } 125 126 // Reset protections to read/write so the area can be reused 127 if (auto EC = sys::Memory::protectMappedMemory( 128 {Base.toPtr<void *>(), Allocations[Base].Size}, 129 sys::Memory::ProtectionFlags::MF_READ | 130 sys::Memory::ProtectionFlags::MF_WRITE)) { 131 AllErr = joinErrors(std::move(AllErr), errorCodeToError(EC)); 132 } 133 134 Allocations.erase(Base); 135 } 136 } 137 138 OnDeinitialized(std::move(AllErr)); 139 } 140 141 void InProcessMemoryMapper::release(ArrayRef<ExecutorAddr> Bases, 142 OnReleasedFunction OnReleased) { 143 Error Err = Error::success(); 144 145 for (auto Base : Bases) { 146 std::vector<ExecutorAddr> AllocAddrs; 147 size_t Size; 148 { 149 std::lock_guard<std::mutex> Lock(Mutex); 150 auto &R = Reservations[Base.toPtr<void *>()]; 151 Size = R.Size; 152 AllocAddrs.swap(R.Allocations); 153 } 154 155 // deinitialize sub allocations 156 std::promise<MSVCPError> P; 157 auto F = P.get_future(); 158 deinitialize(AllocAddrs, [&](Error Err) { P.set_value(std::move(Err)); }); 159 if (Error E = F.get()) { 160 Err = joinErrors(std::move(Err), std::move(E)); 161 } 162 163 // free the memory 164 auto MB = sys::MemoryBlock(Base.toPtr<void *>(), Size); 165 166 auto EC = sys::Memory::releaseMappedMemory(MB); 167 if (EC) { 168 Err = joinErrors(std::move(Err), errorCodeToError(EC)); 169 } 170 171 std::lock_guard<std::mutex> Lock(Mutex); 172 Reservations.erase(Base.toPtr<void *>()); 173 } 174 175 OnReleased(std::move(Err)); 176 } 177 178 InProcessMemoryMapper::~InProcessMemoryMapper() { 179 std::vector<ExecutorAddr> ReservationAddrs; 180 { 181 std::lock_guard<std::mutex> Lock(Mutex); 182 183 ReservationAddrs.reserve(Reservations.size()); 184 for (const auto &R : Reservations) { 185 ReservationAddrs.push_back(ExecutorAddr::fromPtr(R.getFirst())); 186 } 187 } 188 189 std::promise<MSVCPError> P; 190 auto F = P.get_future(); 191 release(ReservationAddrs, [&](Error Err) { P.set_value(std::move(Err)); }); 192 cantFail(F.get()); 193 } 194 195 // SharedMemoryMapper 196 197 SharedMemoryMapper::SharedMemoryMapper(ExecutorProcessControl &EPC, 198 SymbolAddrs SAs, size_t PageSize) 199 : EPC(EPC), SAs(SAs), PageSize(PageSize) { 200 #if (!defined(LLVM_ON_UNIX) || defined(__ANDROID__)) && !defined(_WIN32) 201 llvm_unreachable("SharedMemoryMapper is not supported on this platform yet"); 202 #endif 203 } 204 205 Expected<std::unique_ptr<SharedMemoryMapper>> 206 SharedMemoryMapper::Create(ExecutorProcessControl &EPC, SymbolAddrs SAs) { 207 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32) 208 auto PageSize = sys::Process::getPageSize(); 209 if (!PageSize) 210 return PageSize.takeError(); 211 212 return std::make_unique<SharedMemoryMapper>(EPC, SAs, *PageSize); 213 #else 214 return make_error<StringError>( 215 "SharedMemoryMapper is not supported on this platform yet", 216 inconvertibleErrorCode()); 217 #endif 218 } 219 220 void SharedMemoryMapper::reserve(size_t NumBytes, 221 OnReservedFunction OnReserved) { 222 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32) 223 224 EPC.callSPSWrapperAsync< 225 rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>( 226 SAs.Reserve, 227 [this, NumBytes, OnReserved = std::move(OnReserved)]( 228 Error SerializationErr, 229 Expected<std::pair<ExecutorAddr, std::string>> Result) mutable { 230 if (SerializationErr) { 231 cantFail(Result.takeError()); 232 return OnReserved(std::move(SerializationErr)); 233 } 234 235 if (!Result) 236 return OnReserved(Result.takeError()); 237 238 ExecutorAddr RemoteAddr; 239 std::string SharedMemoryName; 240 std::tie(RemoteAddr, SharedMemoryName) = std::move(*Result); 241 242 void *LocalAddr = nullptr; 243 244 #if defined(LLVM_ON_UNIX) 245 246 #if defined(__MVS__) 247 ArrayRef<uint8_t> Data( 248 reinterpret_cast<const uint8_t *>(SharedMemoryName.c_str()), 249 SharedMemoryName.size()); 250 auto HashedName = BLAKE3::hash<sizeof(key_t)>(Data); 251 key_t Key = *reinterpret_cast<key_t *>(HashedName.data()); 252 int SharedMemoryId = 253 shmget(Key, NumBytes, IPC_CREAT | __IPC_SHAREAS | 0700); 254 if (SharedMemoryId < 0) { 255 return OnReserved(errorCodeToError( 256 std::error_code(errno, std::generic_category()))); 257 } 258 LocalAddr = shmat(SharedMemoryId, nullptr, 0); 259 if (LocalAddr == reinterpret_cast<void *>(-1)) { 260 return OnReserved(errorCodeToError( 261 std::error_code(errno, std::generic_category()))); 262 } 263 #else 264 int SharedMemoryFile = shm_open(SharedMemoryName.c_str(), O_RDWR, 0700); 265 if (SharedMemoryFile < 0) { 266 return OnReserved(errorCodeToError(errnoAsErrorCode())); 267 } 268 269 // this prevents other processes from accessing it by name 270 shm_unlink(SharedMemoryName.c_str()); 271 272 LocalAddr = mmap(nullptr, NumBytes, PROT_READ | PROT_WRITE, MAP_SHARED, 273 SharedMemoryFile, 0); 274 if (LocalAddr == MAP_FAILED) { 275 return OnReserved(errorCodeToError(errnoAsErrorCode())); 276 } 277 278 close(SharedMemoryFile); 279 #endif 280 281 #elif defined(_WIN32) 282 283 std::wstring WideSharedMemoryName(SharedMemoryName.begin(), 284 SharedMemoryName.end()); 285 HANDLE SharedMemoryFile = OpenFileMappingW( 286 FILE_MAP_ALL_ACCESS, FALSE, WideSharedMemoryName.c_str()); 287 if (!SharedMemoryFile) 288 return OnReserved(errorCodeToError(mapWindowsError(GetLastError()))); 289 290 LocalAddr = 291 MapViewOfFile(SharedMemoryFile, FILE_MAP_ALL_ACCESS, 0, 0, 0); 292 if (!LocalAddr) { 293 CloseHandle(SharedMemoryFile); 294 return OnReserved(errorCodeToError(mapWindowsError(GetLastError()))); 295 } 296 297 CloseHandle(SharedMemoryFile); 298 299 #endif 300 { 301 std::lock_guard<std::mutex> Lock(Mutex); 302 Reservations.insert({RemoteAddr, {LocalAddr, NumBytes}}); 303 } 304 305 OnReserved(ExecutorAddrRange(RemoteAddr, NumBytes)); 306 }, 307 SAs.Instance, static_cast<uint64_t>(NumBytes)); 308 309 #else 310 OnReserved(make_error<StringError>( 311 "SharedMemoryMapper is not supported on this platform yet", 312 inconvertibleErrorCode())); 313 #endif 314 } 315 316 char *SharedMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) { 317 auto R = Reservations.upper_bound(Addr); 318 assert(R != Reservations.begin() && "Attempt to prepare unreserved range"); 319 R--; 320 321 ExecutorAddrDiff Offset = Addr - R->first; 322 323 return static_cast<char *>(R->second.LocalAddr) + Offset; 324 } 325 326 void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI, 327 OnInitializedFunction OnInitialized) { 328 auto Reservation = Reservations.upper_bound(AI.MappingBase); 329 assert(Reservation != Reservations.begin() && "Attempt to initialize unreserved range"); 330 Reservation--; 331 332 auto AllocationOffset = AI.MappingBase - Reservation->first; 333 334 tpctypes::SharedMemoryFinalizeRequest FR; 335 336 AI.Actions.swap(FR.Actions); 337 338 FR.Segments.reserve(AI.Segments.size()); 339 340 for (auto Segment : AI.Segments) { 341 char *Base = static_cast<char *>(Reservation->second.LocalAddr) + 342 AllocationOffset + Segment.Offset; 343 std::memset(Base + Segment.ContentSize, 0, Segment.ZeroFillSize); 344 345 tpctypes::SharedMemorySegFinalizeRequest SegReq; 346 SegReq.RAG = {Segment.AG.getMemProt(), 347 Segment.AG.getMemLifetime() == MemLifetime::Finalize}; 348 SegReq.Addr = AI.MappingBase + Segment.Offset; 349 SegReq.Size = Segment.ContentSize + Segment.ZeroFillSize; 350 351 FR.Segments.push_back(SegReq); 352 } 353 354 EPC.callSPSWrapperAsync< 355 rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>( 356 SAs.Initialize, 357 [OnInitialized = std::move(OnInitialized)]( 358 Error SerializationErr, Expected<ExecutorAddr> Result) mutable { 359 if (SerializationErr) { 360 cantFail(Result.takeError()); 361 return OnInitialized(std::move(SerializationErr)); 362 } 363 364 OnInitialized(std::move(Result)); 365 }, 366 SAs.Instance, Reservation->first, std::move(FR)); 367 } 368 369 void SharedMemoryMapper::deinitialize( 370 ArrayRef<ExecutorAddr> Allocations, 371 MemoryMapper::OnDeinitializedFunction OnDeinitialized) { 372 EPC.callSPSWrapperAsync< 373 rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>( 374 SAs.Deinitialize, 375 [OnDeinitialized = std::move(OnDeinitialized)](Error SerializationErr, 376 Error Result) mutable { 377 if (SerializationErr) { 378 cantFail(std::move(Result)); 379 return OnDeinitialized(std::move(SerializationErr)); 380 } 381 382 OnDeinitialized(std::move(Result)); 383 }, 384 SAs.Instance, Allocations); 385 } 386 387 void SharedMemoryMapper::release(ArrayRef<ExecutorAddr> Bases, 388 OnReleasedFunction OnReleased) { 389 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32) 390 Error Err = Error::success(); 391 392 { 393 std::lock_guard<std::mutex> Lock(Mutex); 394 395 for (auto Base : Bases) { 396 397 #if defined(LLVM_ON_UNIX) 398 399 #if defined(__MVS__) 400 if (shmdt(Reservations[Base].LocalAddr) < 0) 401 Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode())); 402 #else 403 if (munmap(Reservations[Base].LocalAddr, Reservations[Base].Size) != 0) 404 Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode())); 405 #endif 406 407 #elif defined(_WIN32) 408 409 if (!UnmapViewOfFile(Reservations[Base].LocalAddr)) 410 Err = joinErrors(std::move(Err), 411 errorCodeToError(mapWindowsError(GetLastError()))); 412 413 #endif 414 415 Reservations.erase(Base); 416 } 417 } 418 419 EPC.callSPSWrapperAsync< 420 rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>( 421 SAs.Release, 422 [OnReleased = std::move(OnReleased), 423 Err = std::move(Err)](Error SerializationErr, Error Result) mutable { 424 if (SerializationErr) { 425 cantFail(std::move(Result)); 426 return OnReleased( 427 joinErrors(std::move(Err), std::move(SerializationErr))); 428 } 429 430 return OnReleased(joinErrors(std::move(Err), std::move(Result))); 431 }, 432 SAs.Instance, Bases); 433 #else 434 OnReleased(make_error<StringError>( 435 "SharedMemoryMapper is not supported on this platform yet", 436 inconvertibleErrorCode())); 437 #endif 438 } 439 440 SharedMemoryMapper::~SharedMemoryMapper() { 441 std::lock_guard<std::mutex> Lock(Mutex); 442 for (const auto &R : Reservations) { 443 444 #if defined(LLVM_ON_UNIX) && !defined(__ANDROID__) 445 446 #if defined(__MVS__) 447 shmdt(R.second.LocalAddr); 448 #else 449 munmap(R.second.LocalAddr, R.second.Size); 450 #endif 451 452 #elif defined(_WIN32) 453 454 UnmapViewOfFile(R.second.LocalAddr); 455 456 #else 457 458 (void)R; 459 460 #endif 461 } 462 } 463 464 } // namespace orc 465 466 } // namespace llvm 467