1 //===- MemoryMapper.cpp - Cross-process memory mapper ------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/MemoryMapper.h" 10 11 #include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX 12 #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" 13 #include "llvm/Support/WindowsError.h" 14 15 #if defined(LLVM_ON_UNIX) && !defined(__ANDROID__) 16 #include <fcntl.h> 17 #include <sys/mman.h> 18 #if defined(__MVS__) 19 #include "llvm/Support/BLAKE3.h" 20 #include <sys/shm.h> 21 #endif 22 #include <unistd.h> 23 #elif defined(_WIN32) 24 #include <windows.h> 25 #endif 26 27 namespace llvm { 28 namespace orc { 29 30 MemoryMapper::~MemoryMapper() {} 31 32 InProcessMemoryMapper::InProcessMemoryMapper(size_t PageSize) 33 : PageSize(PageSize) {} 34 35 Expected<std::unique_ptr<InProcessMemoryMapper>> 36 InProcessMemoryMapper::Create() { 37 auto PageSize = sys::Process::getPageSize(); 38 if (!PageSize) 39 return PageSize.takeError(); 40 return std::make_unique<InProcessMemoryMapper>(*PageSize); 41 } 42 43 void InProcessMemoryMapper::reserve(size_t NumBytes, 44 OnReservedFunction OnReserved) { 45 std::error_code EC; 46 auto MB = sys::Memory::allocateMappedMemory( 47 NumBytes, nullptr, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC); 48 49 if (EC) 50 return OnReserved(errorCodeToError(EC)); 51 52 { 53 std::lock_guard<std::mutex> Lock(Mutex); 54 Reservations[MB.base()].Size = MB.allocatedSize(); 55 } 56 57 OnReserved( 58 ExecutorAddrRange(ExecutorAddr::fromPtr(MB.base()), MB.allocatedSize())); 59 } 60 61 char *InProcessMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) { 62 return Addr.toPtr<char *>(); 63 } 64 65 void InProcessMemoryMapper::initialize(MemoryMapper::AllocInfo &AI, 66 OnInitializedFunction OnInitialized) { 67 ExecutorAddr MinAddr(~0ULL); 68 ExecutorAddr MaxAddr(0); 69 70 // FIXME: Release finalize lifetime segments. 71 for (auto &Segment : AI.Segments) { 72 auto Base = AI.MappingBase + Segment.Offset; 73 auto Size = Segment.ContentSize + Segment.ZeroFillSize; 74 75 if (Base < MinAddr) 76 MinAddr = Base; 77 78 if (Base + Size > MaxAddr) 79 MaxAddr = Base + Size; 80 81 std::memset((Base + Segment.ContentSize).toPtr<void *>(), 0, 82 Segment.ZeroFillSize); 83 84 if (auto EC = sys::Memory::protectMappedMemory( 85 {Base.toPtr<void *>(), Size}, 86 toSysMemoryProtectionFlags(Segment.AG.getMemProt()))) { 87 return OnInitialized(errorCodeToError(EC)); 88 } 89 if ((Segment.AG.getMemProt() & MemProt::Exec) == MemProt::Exec) 90 sys::Memory::InvalidateInstructionCache(Base.toPtr<void *>(), Size); 91 } 92 93 std::vector<shared::WrapperFunctionCall> DeinitializeActions; 94 { 95 std::promise<MSVCPExpected<std::vector<shared::WrapperFunctionCall>>> P; 96 auto F = P.get_future(); 97 shared::runFinalizeActions( 98 AI.Actions, [&](Expected<std::vector<shared::WrapperFunctionCall>> R) { 99 P.set_value(std::move(R)); 100 }); 101 if (auto DeinitializeActionsOrErr = F.get()) 102 DeinitializeActions = std::move(*DeinitializeActionsOrErr); 103 else 104 return OnInitialized(DeinitializeActionsOrErr.takeError()); 105 } 106 107 { 108 std::lock_guard<std::mutex> Lock(Mutex); 109 110 // This is the maximum range whose permission have been possibly modified 111 auto &Alloc = Allocations[MinAddr]; 112 Alloc.Size = MaxAddr - MinAddr; 113 Alloc.DeinitializationActions = std::move(DeinitializeActions); 114 Reservations[AI.MappingBase.toPtr<void *>()].Allocations.push_back(MinAddr); 115 } 116 117 OnInitialized(MinAddr); 118 } 119 120 void InProcessMemoryMapper::deinitialize( 121 ArrayRef<ExecutorAddr> Bases, 122 MemoryMapper::OnDeinitializedFunction OnDeinitialized) { 123 Error AllErr = Error::success(); 124 125 { 126 std::lock_guard<std::mutex> Lock(Mutex); 127 128 for (auto Base : llvm::reverse(Bases)) { 129 130 shared::runDeallocActions( 131 Allocations[Base].DeinitializationActions, [&](Error Err) { 132 AllErr = joinErrors(std::move(AllErr), std::move(Err)); 133 }); 134 135 // Reset protections to read/write so the area can be reused 136 if (auto EC = sys::Memory::protectMappedMemory( 137 {Base.toPtr<void *>(), Allocations[Base].Size}, 138 sys::Memory::ProtectionFlags::MF_READ | 139 sys::Memory::ProtectionFlags::MF_WRITE)) { 140 AllErr = joinErrors(std::move(AllErr), errorCodeToError(EC)); 141 } 142 143 Allocations.erase(Base); 144 } 145 } 146 147 OnDeinitialized(std::move(AllErr)); 148 } 149 150 void InProcessMemoryMapper::release(ArrayRef<ExecutorAddr> Bases, 151 OnReleasedFunction OnReleased) { 152 Error Err = Error::success(); 153 154 for (auto Base : Bases) { 155 std::vector<ExecutorAddr> AllocAddrs; 156 size_t Size; 157 { 158 std::lock_guard<std::mutex> Lock(Mutex); 159 auto &R = Reservations[Base.toPtr<void *>()]; 160 Size = R.Size; 161 AllocAddrs.swap(R.Allocations); 162 } 163 164 // deinitialize sub allocations 165 std::promise<MSVCPError> P; 166 auto F = P.get_future(); 167 deinitialize(AllocAddrs, [&](Error Err) { P.set_value(std::move(Err)); }); 168 if (Error E = F.get()) { 169 Err = joinErrors(std::move(Err), std::move(E)); 170 } 171 172 // free the memory 173 auto MB = sys::MemoryBlock(Base.toPtr<void *>(), Size); 174 175 auto EC = sys::Memory::releaseMappedMemory(MB); 176 if (EC) { 177 Err = joinErrors(std::move(Err), errorCodeToError(EC)); 178 } 179 180 std::lock_guard<std::mutex> Lock(Mutex); 181 Reservations.erase(Base.toPtr<void *>()); 182 } 183 184 OnReleased(std::move(Err)); 185 } 186 187 InProcessMemoryMapper::~InProcessMemoryMapper() { 188 std::vector<ExecutorAddr> ReservationAddrs; 189 { 190 std::lock_guard<std::mutex> Lock(Mutex); 191 192 ReservationAddrs.reserve(Reservations.size()); 193 for (const auto &R : Reservations) { 194 ReservationAddrs.push_back(ExecutorAddr::fromPtr(R.getFirst())); 195 } 196 } 197 198 std::promise<MSVCPError> P; 199 auto F = P.get_future(); 200 release(ReservationAddrs, [&](Error Err) { P.set_value(std::move(Err)); }); 201 cantFail(F.get()); 202 } 203 204 // SharedMemoryMapper 205 206 SharedMemoryMapper::SharedMemoryMapper(ExecutorProcessControl &EPC, 207 SymbolAddrs SAs, size_t PageSize) 208 : EPC(EPC), SAs(SAs), PageSize(PageSize) { 209 #if (!defined(LLVM_ON_UNIX) || defined(__ANDROID__)) && !defined(_WIN32) 210 llvm_unreachable("SharedMemoryMapper is not supported on this platform yet"); 211 #endif 212 } 213 214 Expected<std::unique_ptr<SharedMemoryMapper>> 215 SharedMemoryMapper::Create(ExecutorProcessControl &EPC, SymbolAddrs SAs) { 216 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32) 217 auto PageSize = sys::Process::getPageSize(); 218 if (!PageSize) 219 return PageSize.takeError(); 220 221 return std::make_unique<SharedMemoryMapper>(EPC, SAs, *PageSize); 222 #else 223 return make_error<StringError>( 224 "SharedMemoryMapper is not supported on this platform yet", 225 inconvertibleErrorCode()); 226 #endif 227 } 228 229 void SharedMemoryMapper::reserve(size_t NumBytes, 230 OnReservedFunction OnReserved) { 231 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32) 232 233 int SharedMemoryId = -1; 234 EPC.callSPSWrapperAsync< 235 rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>( 236 SAs.Reserve, 237 [this, NumBytes, OnReserved = std::move(OnReserved), SharedMemoryId]( 238 Error SerializationErr, 239 Expected<std::pair<ExecutorAddr, std::string>> Result) mutable { 240 if (SerializationErr) { 241 cantFail(Result.takeError()); 242 return OnReserved(std::move(SerializationErr)); 243 } 244 245 if (!Result) 246 return OnReserved(Result.takeError()); 247 248 ExecutorAddr RemoteAddr; 249 std::string SharedMemoryName; 250 std::tie(RemoteAddr, SharedMemoryName) = std::move(*Result); 251 252 void *LocalAddr = nullptr; 253 254 #if defined(LLVM_ON_UNIX) 255 256 #if defined(__MVS__) 257 ArrayRef<uint8_t> Data( 258 reinterpret_cast<const uint8_t *>(SharedMemoryName.c_str()), 259 SharedMemoryName.size()); 260 auto HashedName = BLAKE3::hash<sizeof(key_t)>(Data); 261 key_t Key = *reinterpret_cast<key_t *>(HashedName.data()); 262 SharedMemoryId = 263 shmget(Key, NumBytes, IPC_CREAT | __IPC_SHAREAS | 0700); 264 if (SharedMemoryId < 0) { 265 return OnReserved(errorCodeToError( 266 std::error_code(errno, std::generic_category()))); 267 } 268 LocalAddr = shmat(SharedMemoryId, nullptr, 0); 269 if (LocalAddr == reinterpret_cast<void *>(-1)) { 270 return OnReserved(errorCodeToError( 271 std::error_code(errno, std::generic_category()))); 272 } 273 #else 274 int SharedMemoryFile = shm_open(SharedMemoryName.c_str(), O_RDWR, 0700); 275 if (SharedMemoryFile < 0) { 276 return OnReserved(errorCodeToError(errnoAsErrorCode())); 277 } 278 279 // this prevents other processes from accessing it by name 280 shm_unlink(SharedMemoryName.c_str()); 281 282 LocalAddr = mmap(nullptr, NumBytes, PROT_READ | PROT_WRITE, MAP_SHARED, 283 SharedMemoryFile, 0); 284 if (LocalAddr == MAP_FAILED) { 285 return OnReserved(errorCodeToError(errnoAsErrorCode())); 286 } 287 288 close(SharedMemoryFile); 289 #endif 290 291 #elif defined(_WIN32) 292 293 std::wstring WideSharedMemoryName(SharedMemoryName.begin(), 294 SharedMemoryName.end()); 295 HANDLE SharedMemoryFile = OpenFileMappingW( 296 FILE_MAP_ALL_ACCESS, FALSE, WideSharedMemoryName.c_str()); 297 if (!SharedMemoryFile) 298 return OnReserved(errorCodeToError(mapWindowsError(GetLastError()))); 299 300 LocalAddr = 301 MapViewOfFile(SharedMemoryFile, FILE_MAP_ALL_ACCESS, 0, 0, 0); 302 if (!LocalAddr) { 303 CloseHandle(SharedMemoryFile); 304 return OnReserved(errorCodeToError(mapWindowsError(GetLastError()))); 305 } 306 307 CloseHandle(SharedMemoryFile); 308 309 #endif 310 { 311 std::lock_guard<std::mutex> Lock(Mutex); 312 Reservations.insert( 313 {RemoteAddr, {LocalAddr, NumBytes, SharedMemoryId}}); 314 } 315 316 OnReserved(ExecutorAddrRange(RemoteAddr, NumBytes)); 317 }, 318 SAs.Instance, static_cast<uint64_t>(NumBytes)); 319 320 #else 321 OnReserved(make_error<StringError>( 322 "SharedMemoryMapper is not supported on this platform yet", 323 inconvertibleErrorCode())); 324 #endif 325 } 326 327 char *SharedMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) { 328 auto R = Reservations.upper_bound(Addr); 329 assert(R != Reservations.begin() && "Attempt to prepare unreserved range"); 330 R--; 331 332 ExecutorAddrDiff Offset = Addr - R->first; 333 334 return static_cast<char *>(R->second.LocalAddr) + Offset; 335 } 336 337 void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI, 338 OnInitializedFunction OnInitialized) { 339 auto Reservation = Reservations.upper_bound(AI.MappingBase); 340 assert(Reservation != Reservations.begin() && "Attempt to initialize unreserved range"); 341 Reservation--; 342 343 auto AllocationOffset = AI.MappingBase - Reservation->first; 344 345 tpctypes::SharedMemoryFinalizeRequest FR; 346 347 AI.Actions.swap(FR.Actions); 348 349 FR.Segments.reserve(AI.Segments.size()); 350 351 for (auto Segment : AI.Segments) { 352 char *Base = static_cast<char *>(Reservation->second.LocalAddr) + 353 AllocationOffset + Segment.Offset; 354 std::memset(Base + Segment.ContentSize, 0, Segment.ZeroFillSize); 355 356 tpctypes::SharedMemorySegFinalizeRequest SegReq; 357 SegReq.RAG = {Segment.AG.getMemProt(), 358 Segment.AG.getMemLifetime() == MemLifetime::Finalize}; 359 SegReq.Addr = AI.MappingBase + Segment.Offset; 360 SegReq.Size = Segment.ContentSize + Segment.ZeroFillSize; 361 362 FR.Segments.push_back(SegReq); 363 } 364 365 EPC.callSPSWrapperAsync< 366 rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>( 367 SAs.Initialize, 368 [OnInitialized = std::move(OnInitialized)]( 369 Error SerializationErr, Expected<ExecutorAddr> Result) mutable { 370 if (SerializationErr) { 371 cantFail(Result.takeError()); 372 return OnInitialized(std::move(SerializationErr)); 373 } 374 375 OnInitialized(std::move(Result)); 376 }, 377 SAs.Instance, Reservation->first, std::move(FR)); 378 } 379 380 void SharedMemoryMapper::deinitialize( 381 ArrayRef<ExecutorAddr> Allocations, 382 MemoryMapper::OnDeinitializedFunction OnDeinitialized) { 383 EPC.callSPSWrapperAsync< 384 rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>( 385 SAs.Deinitialize, 386 [OnDeinitialized = std::move(OnDeinitialized)](Error SerializationErr, 387 Error Result) mutable { 388 if (SerializationErr) { 389 cantFail(std::move(Result)); 390 return OnDeinitialized(std::move(SerializationErr)); 391 } 392 393 OnDeinitialized(std::move(Result)); 394 }, 395 SAs.Instance, Allocations); 396 } 397 398 void SharedMemoryMapper::release(ArrayRef<ExecutorAddr> Bases, 399 OnReleasedFunction OnReleased) { 400 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32) 401 Error Err = Error::success(); 402 403 { 404 std::lock_guard<std::mutex> Lock(Mutex); 405 406 for (auto Base : Bases) { 407 408 #if defined(LLVM_ON_UNIX) 409 410 #if defined(__MVS__) 411 if (shmdt(Reservations[Base].LocalAddr) < 0 || 412 shmctl(Reservations[Base].SharedMemoryId, IPC_RMID, NULL) < 0) 413 Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode())); 414 #else 415 if (munmap(Reservations[Base].LocalAddr, Reservations[Base].Size) != 0) 416 Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode())); 417 #endif 418 419 #elif defined(_WIN32) 420 421 if (!UnmapViewOfFile(Reservations[Base].LocalAddr)) 422 Err = joinErrors(std::move(Err), 423 errorCodeToError(mapWindowsError(GetLastError()))); 424 425 #endif 426 427 Reservations.erase(Base); 428 } 429 } 430 431 EPC.callSPSWrapperAsync< 432 rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>( 433 SAs.Release, 434 [OnReleased = std::move(OnReleased), 435 Err = std::move(Err)](Error SerializationErr, Error Result) mutable { 436 if (SerializationErr) { 437 cantFail(std::move(Result)); 438 return OnReleased( 439 joinErrors(std::move(Err), std::move(SerializationErr))); 440 } 441 442 return OnReleased(joinErrors(std::move(Err), std::move(Result))); 443 }, 444 SAs.Instance, Bases); 445 #else 446 OnReleased(make_error<StringError>( 447 "SharedMemoryMapper is not supported on this platform yet", 448 inconvertibleErrorCode())); 449 #endif 450 } 451 452 SharedMemoryMapper::~SharedMemoryMapper() { 453 std::lock_guard<std::mutex> Lock(Mutex); 454 for (const auto &R : Reservations) { 455 456 #if defined(LLVM_ON_UNIX) && !defined(__ANDROID__) 457 458 #if defined(__MVS__) 459 shmdt(R.second.LocalAddr); 460 #else 461 munmap(R.second.LocalAddr, R.second.Size); 462 #endif 463 464 #elif defined(_WIN32) 465 466 UnmapViewOfFile(R.second.LocalAddr); 467 468 #else 469 470 (void)R; 471 472 #endif 473 } 474 } 475 476 } // namespace orc 477 478 } // namespace llvm 479