1 //===- MemoryMapper.cpp - Cross-process memory mapper ------------*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ExecutionEngine/Orc/MemoryMapper.h"
10
11 #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
12 #include "llvm/Support/WindowsError.h"
13
14 #include <algorithm>
15
16 #if defined(LLVM_ON_UNIX) && !defined(__ANDROID__)
17 #include <fcntl.h>
18 #include <sys/mman.h>
19 #if defined(__MVS__)
20 #include "llvm/Support/BLAKE3.h"
21 #include <sys/shm.h>
22 #endif
23 #include <unistd.h>
24 #elif defined(_WIN32)
25 #include <windows.h>
26 #endif
27
28 namespace llvm {
29 namespace orc {
30
~MemoryMapper()31 MemoryMapper::~MemoryMapper() {}
32
InProcessMemoryMapper(size_t PageSize)33 InProcessMemoryMapper::InProcessMemoryMapper(size_t PageSize)
34 : PageSize(PageSize) {}
35
36 Expected<std::unique_ptr<InProcessMemoryMapper>>
Create()37 InProcessMemoryMapper::Create() {
38 auto PageSize = sys::Process::getPageSize();
39 if (!PageSize)
40 return PageSize.takeError();
41 return std::make_unique<InProcessMemoryMapper>(*PageSize);
42 }
43
reserve(size_t NumBytes,OnReservedFunction OnReserved)44 void InProcessMemoryMapper::reserve(size_t NumBytes,
45 OnReservedFunction OnReserved) {
46 std::error_code EC;
47 auto MB = sys::Memory::allocateMappedMemory(
48 NumBytes, nullptr, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC);
49
50 if (EC)
51 return OnReserved(errorCodeToError(EC));
52
53 {
54 std::lock_guard<std::mutex> Lock(Mutex);
55 Reservations[MB.base()].Size = MB.allocatedSize();
56 }
57
58 OnReserved(
59 ExecutorAddrRange(ExecutorAddr::fromPtr(MB.base()), MB.allocatedSize()));
60 }
61
prepare(ExecutorAddr Addr,size_t ContentSize)62 char *InProcessMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) {
63 return Addr.toPtr<char *>();
64 }
65
initialize(MemoryMapper::AllocInfo & AI,OnInitializedFunction OnInitialized)66 void InProcessMemoryMapper::initialize(MemoryMapper::AllocInfo &AI,
67 OnInitializedFunction OnInitialized) {
68 ExecutorAddr MinAddr(~0ULL);
69 ExecutorAddr MaxAddr(0);
70
71 // FIXME: Release finalize lifetime segments.
72 for (auto &Segment : AI.Segments) {
73 auto Base = AI.MappingBase + Segment.Offset;
74 auto Size = Segment.ContentSize + Segment.ZeroFillSize;
75
76 if (Base < MinAddr)
77 MinAddr = Base;
78
79 if (Base + Size > MaxAddr)
80 MaxAddr = Base + Size;
81
82 std::memset((Base + Segment.ContentSize).toPtr<void *>(), 0,
83 Segment.ZeroFillSize);
84
85 if (auto EC = sys::Memory::protectMappedMemory(
86 {Base.toPtr<void *>(), Size},
87 toSysMemoryProtectionFlags(Segment.AG.getMemProt()))) {
88 return OnInitialized(errorCodeToError(EC));
89 }
90 if ((Segment.AG.getMemProt() & MemProt::Exec) == MemProt::Exec)
91 sys::Memory::InvalidateInstructionCache(Base.toPtr<void *>(), Size);
92 }
93
94 auto DeinitializeActions = shared::runFinalizeActions(AI.Actions);
95 if (!DeinitializeActions)
96 return OnInitialized(DeinitializeActions.takeError());
97
98 {
99 std::lock_guard<std::mutex> Lock(Mutex);
100
101 // This is the maximum range whose permission have been possibly modified
102 Allocations[MinAddr].Size = MaxAddr - MinAddr;
103 Allocations[MinAddr].DeinitializationActions =
104 std::move(*DeinitializeActions);
105 Reservations[AI.MappingBase.toPtr<void *>()].Allocations.push_back(MinAddr);
106 }
107
108 OnInitialized(MinAddr);
109 }
110
deinitialize(ArrayRef<ExecutorAddr> Bases,MemoryMapper::OnDeinitializedFunction OnDeinitialized)111 void InProcessMemoryMapper::deinitialize(
112 ArrayRef<ExecutorAddr> Bases,
113 MemoryMapper::OnDeinitializedFunction OnDeinitialized) {
114 Error AllErr = Error::success();
115
116 {
117 std::lock_guard<std::mutex> Lock(Mutex);
118
119 for (auto Base : llvm::reverse(Bases)) {
120
121 if (Error Err = shared::runDeallocActions(
122 Allocations[Base].DeinitializationActions)) {
123 AllErr = joinErrors(std::move(AllErr), std::move(Err));
124 }
125
126 // Reset protections to read/write so the area can be reused
127 if (auto EC = sys::Memory::protectMappedMemory(
128 {Base.toPtr<void *>(), Allocations[Base].Size},
129 sys::Memory::ProtectionFlags::MF_READ |
130 sys::Memory::ProtectionFlags::MF_WRITE)) {
131 AllErr = joinErrors(std::move(AllErr), errorCodeToError(EC));
132 }
133
134 Allocations.erase(Base);
135 }
136 }
137
138 OnDeinitialized(std::move(AllErr));
139 }
140
release(ArrayRef<ExecutorAddr> Bases,OnReleasedFunction OnReleased)141 void InProcessMemoryMapper::release(ArrayRef<ExecutorAddr> Bases,
142 OnReleasedFunction OnReleased) {
143 Error Err = Error::success();
144
145 for (auto Base : Bases) {
146 std::vector<ExecutorAddr> AllocAddrs;
147 size_t Size;
148 {
149 std::lock_guard<std::mutex> Lock(Mutex);
150 auto &R = Reservations[Base.toPtr<void *>()];
151 Size = R.Size;
152 AllocAddrs.swap(R.Allocations);
153 }
154
155 // deinitialize sub allocations
156 std::promise<MSVCPError> P;
157 auto F = P.get_future();
158 deinitialize(AllocAddrs, [&](Error Err) { P.set_value(std::move(Err)); });
159 if (Error E = F.get()) {
160 Err = joinErrors(std::move(Err), std::move(E));
161 }
162
163 // free the memory
164 auto MB = sys::MemoryBlock(Base.toPtr<void *>(), Size);
165
166 auto EC = sys::Memory::releaseMappedMemory(MB);
167 if (EC) {
168 Err = joinErrors(std::move(Err), errorCodeToError(EC));
169 }
170
171 std::lock_guard<std::mutex> Lock(Mutex);
172 Reservations.erase(Base.toPtr<void *>());
173 }
174
175 OnReleased(std::move(Err));
176 }
177
~InProcessMemoryMapper()178 InProcessMemoryMapper::~InProcessMemoryMapper() {
179 std::vector<ExecutorAddr> ReservationAddrs;
180 {
181 std::lock_guard<std::mutex> Lock(Mutex);
182
183 ReservationAddrs.reserve(Reservations.size());
184 for (const auto &R : Reservations) {
185 ReservationAddrs.push_back(ExecutorAddr::fromPtr(R.getFirst()));
186 }
187 }
188
189 std::promise<MSVCPError> P;
190 auto F = P.get_future();
191 release(ReservationAddrs, [&](Error Err) { P.set_value(std::move(Err)); });
192 cantFail(F.get());
193 }
194
195 // SharedMemoryMapper
196
SharedMemoryMapper(ExecutorProcessControl & EPC,SymbolAddrs SAs,size_t PageSize)197 SharedMemoryMapper::SharedMemoryMapper(ExecutorProcessControl &EPC,
198 SymbolAddrs SAs, size_t PageSize)
199 : EPC(EPC), SAs(SAs), PageSize(PageSize) {
200 #if (!defined(LLVM_ON_UNIX) || defined(__ANDROID__)) && !defined(_WIN32)
201 llvm_unreachable("SharedMemoryMapper is not supported on this platform yet");
202 #endif
203 }
204
205 Expected<std::unique_ptr<SharedMemoryMapper>>
Create(ExecutorProcessControl & EPC,SymbolAddrs SAs)206 SharedMemoryMapper::Create(ExecutorProcessControl &EPC, SymbolAddrs SAs) {
207 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
208 auto PageSize = sys::Process::getPageSize();
209 if (!PageSize)
210 return PageSize.takeError();
211
212 return std::make_unique<SharedMemoryMapper>(EPC, SAs, *PageSize);
213 #else
214 return make_error<StringError>(
215 "SharedMemoryMapper is not supported on this platform yet",
216 inconvertibleErrorCode());
217 #endif
218 }
219
reserve(size_t NumBytes,OnReservedFunction OnReserved)220 void SharedMemoryMapper::reserve(size_t NumBytes,
221 OnReservedFunction OnReserved) {
222 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
223
224 EPC.callSPSWrapperAsync<
225 rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>(
226 SAs.Reserve,
227 [this, NumBytes, OnReserved = std::move(OnReserved)](
228 Error SerializationErr,
229 Expected<std::pair<ExecutorAddr, std::string>> Result) mutable {
230 if (SerializationErr) {
231 cantFail(Result.takeError());
232 return OnReserved(std::move(SerializationErr));
233 }
234
235 if (!Result)
236 return OnReserved(Result.takeError());
237
238 ExecutorAddr RemoteAddr;
239 std::string SharedMemoryName;
240 std::tie(RemoteAddr, SharedMemoryName) = std::move(*Result);
241
242 void *LocalAddr = nullptr;
243
244 #if defined(LLVM_ON_UNIX)
245
246 #if defined(__MVS__)
247 ArrayRef<uint8_t> Data(
248 reinterpret_cast<const uint8_t *>(SharedMemoryName.c_str()),
249 SharedMemoryName.size());
250 auto HashedName = BLAKE3::hash<sizeof(key_t)>(Data);
251 key_t Key = *reinterpret_cast<key_t *>(HashedName.data());
252 int SharedMemoryId =
253 shmget(Key, NumBytes, IPC_CREAT | __IPC_SHAREAS | 0700);
254 if (SharedMemoryId < 0) {
255 return OnReserved(errorCodeToError(
256 std::error_code(errno, std::generic_category())));
257 }
258 LocalAddr = shmat(SharedMemoryId, nullptr, 0);
259 if (LocalAddr == reinterpret_cast<void *>(-1)) {
260 return OnReserved(errorCodeToError(
261 std::error_code(errno, std::generic_category())));
262 }
263 #else
264 int SharedMemoryFile = shm_open(SharedMemoryName.c_str(), O_RDWR, 0700);
265 if (SharedMemoryFile < 0) {
266 return OnReserved(errorCodeToError(errnoAsErrorCode()));
267 }
268
269 // this prevents other processes from accessing it by name
270 shm_unlink(SharedMemoryName.c_str());
271
272 LocalAddr = mmap(nullptr, NumBytes, PROT_READ | PROT_WRITE, MAP_SHARED,
273 SharedMemoryFile, 0);
274 if (LocalAddr == MAP_FAILED) {
275 return OnReserved(errorCodeToError(errnoAsErrorCode()));
276 }
277
278 close(SharedMemoryFile);
279 #endif
280
281 #elif defined(_WIN32)
282
283 std::wstring WideSharedMemoryName(SharedMemoryName.begin(),
284 SharedMemoryName.end());
285 HANDLE SharedMemoryFile = OpenFileMappingW(
286 FILE_MAP_ALL_ACCESS, FALSE, WideSharedMemoryName.c_str());
287 if (!SharedMemoryFile)
288 return OnReserved(errorCodeToError(mapWindowsError(GetLastError())));
289
290 LocalAddr =
291 MapViewOfFile(SharedMemoryFile, FILE_MAP_ALL_ACCESS, 0, 0, 0);
292 if (!LocalAddr) {
293 CloseHandle(SharedMemoryFile);
294 return OnReserved(errorCodeToError(mapWindowsError(GetLastError())));
295 }
296
297 CloseHandle(SharedMemoryFile);
298
299 #endif
300 {
301 std::lock_guard<std::mutex> Lock(Mutex);
302 Reservations.insert({RemoteAddr, {LocalAddr, NumBytes}});
303 }
304
305 OnReserved(ExecutorAddrRange(RemoteAddr, NumBytes));
306 },
307 SAs.Instance, static_cast<uint64_t>(NumBytes));
308
309 #else
310 OnReserved(make_error<StringError>(
311 "SharedMemoryMapper is not supported on this platform yet",
312 inconvertibleErrorCode()));
313 #endif
314 }
315
prepare(ExecutorAddr Addr,size_t ContentSize)316 char *SharedMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) {
317 auto R = Reservations.upper_bound(Addr);
318 assert(R != Reservations.begin() && "Attempt to prepare unreserved range");
319 R--;
320
321 ExecutorAddrDiff Offset = Addr - R->first;
322
323 return static_cast<char *>(R->second.LocalAddr) + Offset;
324 }
325
initialize(MemoryMapper::AllocInfo & AI,OnInitializedFunction OnInitialized)326 void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI,
327 OnInitializedFunction OnInitialized) {
328 auto Reservation = Reservations.upper_bound(AI.MappingBase);
329 assert(Reservation != Reservations.begin() && "Attempt to initialize unreserved range");
330 Reservation--;
331
332 auto AllocationOffset = AI.MappingBase - Reservation->first;
333
334 tpctypes::SharedMemoryFinalizeRequest FR;
335
336 AI.Actions.swap(FR.Actions);
337
338 FR.Segments.reserve(AI.Segments.size());
339
340 for (auto Segment : AI.Segments) {
341 char *Base = static_cast<char *>(Reservation->second.LocalAddr) +
342 AllocationOffset + Segment.Offset;
343 std::memset(Base + Segment.ContentSize, 0, Segment.ZeroFillSize);
344
345 tpctypes::SharedMemorySegFinalizeRequest SegReq;
346 SegReq.RAG = {Segment.AG.getMemProt(),
347 Segment.AG.getMemLifetime() == MemLifetime::Finalize};
348 SegReq.Addr = AI.MappingBase + Segment.Offset;
349 SegReq.Size = Segment.ContentSize + Segment.ZeroFillSize;
350
351 FR.Segments.push_back(SegReq);
352 }
353
354 EPC.callSPSWrapperAsync<
355 rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>(
356 SAs.Initialize,
357 [OnInitialized = std::move(OnInitialized)](
358 Error SerializationErr, Expected<ExecutorAddr> Result) mutable {
359 if (SerializationErr) {
360 cantFail(Result.takeError());
361 return OnInitialized(std::move(SerializationErr));
362 }
363
364 OnInitialized(std::move(Result));
365 },
366 SAs.Instance, Reservation->first, std::move(FR));
367 }
368
deinitialize(ArrayRef<ExecutorAddr> Allocations,MemoryMapper::OnDeinitializedFunction OnDeinitialized)369 void SharedMemoryMapper::deinitialize(
370 ArrayRef<ExecutorAddr> Allocations,
371 MemoryMapper::OnDeinitializedFunction OnDeinitialized) {
372 EPC.callSPSWrapperAsync<
373 rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>(
374 SAs.Deinitialize,
375 [OnDeinitialized = std::move(OnDeinitialized)](Error SerializationErr,
376 Error Result) mutable {
377 if (SerializationErr) {
378 cantFail(std::move(Result));
379 return OnDeinitialized(std::move(SerializationErr));
380 }
381
382 OnDeinitialized(std::move(Result));
383 },
384 SAs.Instance, Allocations);
385 }
386
release(ArrayRef<ExecutorAddr> Bases,OnReleasedFunction OnReleased)387 void SharedMemoryMapper::release(ArrayRef<ExecutorAddr> Bases,
388 OnReleasedFunction OnReleased) {
389 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
390 Error Err = Error::success();
391
392 {
393 std::lock_guard<std::mutex> Lock(Mutex);
394
395 for (auto Base : Bases) {
396
397 #if defined(LLVM_ON_UNIX)
398
399 #if defined(__MVS__)
400 if (shmdt(Reservations[Base].LocalAddr) < 0)
401 Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));
402 #else
403 if (munmap(Reservations[Base].LocalAddr, Reservations[Base].Size) != 0)
404 Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));
405 #endif
406
407 #elif defined(_WIN32)
408
409 if (!UnmapViewOfFile(Reservations[Base].LocalAddr))
410 Err = joinErrors(std::move(Err),
411 errorCodeToError(mapWindowsError(GetLastError())));
412
413 #endif
414
415 Reservations.erase(Base);
416 }
417 }
418
419 EPC.callSPSWrapperAsync<
420 rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>(
421 SAs.Release,
422 [OnReleased = std::move(OnReleased),
423 Err = std::move(Err)](Error SerializationErr, Error Result) mutable {
424 if (SerializationErr) {
425 cantFail(std::move(Result));
426 return OnReleased(
427 joinErrors(std::move(Err), std::move(SerializationErr)));
428 }
429
430 return OnReleased(joinErrors(std::move(Err), std::move(Result)));
431 },
432 SAs.Instance, Bases);
433 #else
434 OnReleased(make_error<StringError>(
435 "SharedMemoryMapper is not supported on this platform yet",
436 inconvertibleErrorCode()));
437 #endif
438 }
439
~SharedMemoryMapper()440 SharedMemoryMapper::~SharedMemoryMapper() {
441 std::lock_guard<std::mutex> Lock(Mutex);
442 for (const auto &R : Reservations) {
443
444 #if defined(LLVM_ON_UNIX) && !defined(__ANDROID__)
445
446 #if defined(__MVS__)
447 shmdt(R.second.LocalAddr);
448 #else
449 munmap(R.second.LocalAddr, R.second.Size);
450 #endif
451
452 #elif defined(_WIN32)
453
454 UnmapViewOfFile(R.second.LocalAddr);
455
456 #else
457
458 (void)R;
459
460 #endif
461 }
462 }
463
464 } // namespace orc
465
466 } // namespace llvm
467