1 //===---------- ExecutorSharedMemoryMapperService.cpp -----------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h"
10
11 #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
12 #include "llvm/Support/Process.h"
13 #include "llvm/Support/WindowsError.h"
14
15 #include <sstream>
16
17 #if defined(LLVM_ON_UNIX)
18 #include <errno.h>
19 #include <fcntl.h>
20 #include <sys/mman.h>
21 #if defined(__MVS__)
22 #include "llvm/Support/BLAKE3.h"
23 #include <sys/shm.h>
24 #endif
25 #include <unistd.h>
26 #endif
27
28 namespace llvm {
29 namespace orc {
30 namespace rt_bootstrap {
31
32 #if defined(_WIN32)
getWindowsProtectionFlags(MemProt MP)33 static DWORD getWindowsProtectionFlags(MemProt MP) {
34 if (MP == MemProt::Read)
35 return PAGE_READONLY;
36 if (MP == MemProt::Write ||
37 MP == (MemProt::Write | MemProt::Read)) {
38 // Note: PAGE_WRITE is not supported by VirtualProtect
39 return PAGE_READWRITE;
40 }
41 if (MP == (MemProt::Read | MemProt::Exec))
42 return PAGE_EXECUTE_READ;
43 if (MP == (MemProt::Read | MemProt::Write | MemProt::Exec))
44 return PAGE_EXECUTE_READWRITE;
45 if (MP == MemProt::Exec)
46 return PAGE_EXECUTE;
47
48 return PAGE_NOACCESS;
49 }
50 #endif
51
52 Expected<std::pair<ExecutorAddr, std::string>>
reserve(uint64_t Size)53 ExecutorSharedMemoryMapperService::reserve(uint64_t Size) {
54 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
55
56 #if defined(LLVM_ON_UNIX)
57
58 std::string SharedMemoryName;
59 {
60 std::stringstream SharedMemoryNameStream;
61 SharedMemoryNameStream << "/jitlink_" << sys::Process::getProcessId() << '_'
62 << (++SharedMemoryCount);
63 SharedMemoryName = SharedMemoryNameStream.str();
64 }
65
66 #if defined(__MVS__)
67 ArrayRef<uint8_t> Data(
68 reinterpret_cast<const uint8_t *>(SharedMemoryName.c_str()),
69 SharedMemoryName.size());
70 auto HashedName = BLAKE3::hash<sizeof(key_t)>(Data);
71 key_t Key = *reinterpret_cast<key_t *>(HashedName.data());
72 int SharedMemoryId =
73 shmget(Key, Size, IPC_CREAT | IPC_EXCL | __IPC_SHAREAS | 0700);
74 if (SharedMemoryId < 0)
75 return errorCodeToError(errnoAsErrorCode());
76
77 void *Addr = shmat(SharedMemoryId, nullptr, 0);
78 if (Addr == reinterpret_cast<void *>(-1))
79 return errorCodeToError(errnoAsErrorCode());
80 #else
81 int SharedMemoryFile =
82 shm_open(SharedMemoryName.c_str(), O_RDWR | O_CREAT | O_EXCL, 0700);
83 if (SharedMemoryFile < 0)
84 return errorCodeToError(errnoAsErrorCode());
85
86 // by default size is 0
87 if (ftruncate(SharedMemoryFile, Size) < 0)
88 return errorCodeToError(errnoAsErrorCode());
89
90 void *Addr = mmap(nullptr, Size, PROT_NONE, MAP_SHARED, SharedMemoryFile, 0);
91 if (Addr == MAP_FAILED)
92 return errorCodeToError(errnoAsErrorCode());
93
94 close(SharedMemoryFile);
95 #endif
96
97 #elif defined(_WIN32)
98
99 std::string SharedMemoryName;
100 {
101 std::stringstream SharedMemoryNameStream;
102 SharedMemoryNameStream << "jitlink_" << sys::Process::getProcessId() << '_'
103 << (++SharedMemoryCount);
104 SharedMemoryName = SharedMemoryNameStream.str();
105 }
106
107 std::wstring WideSharedMemoryName(SharedMemoryName.begin(),
108 SharedMemoryName.end());
109 HANDLE SharedMemoryFile = CreateFileMappingW(
110 INVALID_HANDLE_VALUE, NULL, PAGE_EXECUTE_READWRITE, Size >> 32,
111 Size & 0xffffffff, WideSharedMemoryName.c_str());
112 if (!SharedMemoryFile)
113 return errorCodeToError(mapWindowsError(GetLastError()));
114
115 void *Addr = MapViewOfFile(SharedMemoryFile,
116 FILE_MAP_ALL_ACCESS | FILE_MAP_EXECUTE, 0, 0, 0);
117 if (!Addr) {
118 CloseHandle(SharedMemoryFile);
119 return errorCodeToError(mapWindowsError(GetLastError()));
120 }
121
122 #endif
123
124 {
125 std::lock_guard<std::mutex> Lock(Mutex);
126 Reservations[Addr].Size = Size;
127 #if defined(_WIN32)
128 Reservations[Addr].SharedMemoryFile = SharedMemoryFile;
129 #endif
130 }
131
132 return std::make_pair(ExecutorAddr::fromPtr(Addr),
133 std::move(SharedMemoryName));
134 #else
135 return make_error<StringError>(
136 "SharedMemoryMapper is not supported on this platform yet",
137 inconvertibleErrorCode());
138 #endif
139 }
140
initialize(ExecutorAddr Reservation,tpctypes::SharedMemoryFinalizeRequest & FR)141 Expected<ExecutorAddr> ExecutorSharedMemoryMapperService::initialize(
142 ExecutorAddr Reservation, tpctypes::SharedMemoryFinalizeRequest &FR) {
143 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
144
145 ExecutorAddr MinAddr(~0ULL);
146
147 // Contents are already in place
148 for (auto &Segment : FR.Segments) {
149 if (Segment.Addr < MinAddr)
150 MinAddr = Segment.Addr;
151
152 #if defined(LLVM_ON_UNIX)
153
154 #if defined(__MVS__)
155 // TODO Is it possible to change the protection level?
156 #else
157 int NativeProt = 0;
158 if ((Segment.RAG.Prot & MemProt::Read) == MemProt::Read)
159 NativeProt |= PROT_READ;
160 if ((Segment.RAG.Prot & MemProt::Write) == MemProt::Write)
161 NativeProt |= PROT_WRITE;
162 if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec)
163 NativeProt |= PROT_EXEC;
164
165 if (mprotect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt))
166 return errorCodeToError(errnoAsErrorCode());
167 #endif
168
169 #elif defined(_WIN32)
170
171 DWORD NativeProt = getWindowsProtectionFlags(Segment.RAG.Prot);
172
173 if (!VirtualProtect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt,
174 &NativeProt))
175 return errorCodeToError(mapWindowsError(GetLastError()));
176
177 #endif
178
179 if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec)
180 sys::Memory::InvalidateInstructionCache(Segment.Addr.toPtr<void *>(),
181 Segment.Size);
182 }
183
184 // Run finalization actions and get deinitlization action list.
185 auto DeinitializeActions = shared::runFinalizeActions(FR.Actions);
186 if (!DeinitializeActions) {
187 return DeinitializeActions.takeError();
188 }
189
190 {
191 std::lock_guard<std::mutex> Lock(Mutex);
192 Allocations[MinAddr].DeinitializationActions =
193 std::move(*DeinitializeActions);
194 Reservations[Reservation.toPtr<void *>()].Allocations.push_back(MinAddr);
195 }
196
197 return MinAddr;
198
199 #else
200 return make_error<StringError>(
201 "SharedMemoryMapper is not supported on this platform yet",
202 inconvertibleErrorCode());
203 #endif
204 }
205
deinitialize(const std::vector<ExecutorAddr> & Bases)206 Error ExecutorSharedMemoryMapperService::deinitialize(
207 const std::vector<ExecutorAddr> &Bases) {
208 Error AllErr = Error::success();
209
210 {
211 std::lock_guard<std::mutex> Lock(Mutex);
212
213 for (auto Base : llvm::reverse(Bases)) {
214 if (Error Err = shared::runDeallocActions(
215 Allocations[Base].DeinitializationActions)) {
216 AllErr = joinErrors(std::move(AllErr), std::move(Err));
217 }
218
219 // Remove the allocation from the allocation list of its reservation
220 for (auto &Reservation : Reservations) {
221 auto AllocationIt = llvm::find(Reservation.second.Allocations, Base);
222 if (AllocationIt != Reservation.second.Allocations.end()) {
223 Reservation.second.Allocations.erase(AllocationIt);
224 break;
225 }
226 }
227
228 Allocations.erase(Base);
229 }
230 }
231
232 return AllErr;
233 }
234
release(const std::vector<ExecutorAddr> & Bases)235 Error ExecutorSharedMemoryMapperService::release(
236 const std::vector<ExecutorAddr> &Bases) {
237 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
238 Error Err = Error::success();
239
240 for (auto Base : Bases) {
241 std::vector<ExecutorAddr> AllocAddrs;
242 size_t Size;
243
244 #if defined(_WIN32)
245 HANDLE SharedMemoryFile;
246 #endif
247
248 {
249 std::lock_guard<std::mutex> Lock(Mutex);
250 auto &R = Reservations[Base.toPtr<void *>()];
251 Size = R.Size;
252
253 #if defined(_WIN32)
254 SharedMemoryFile = R.SharedMemoryFile;
255 #endif
256
257 AllocAddrs.swap(R.Allocations);
258 }
259
260 // deinitialize sub allocations
261 if (Error E = deinitialize(AllocAddrs))
262 Err = joinErrors(std::move(Err), std::move(E));
263
264 #if defined(LLVM_ON_UNIX)
265
266 #if defined(__MVS__)
267 (void)Size;
268
269 if (shmdt(Base.toPtr<void *>()) < 0)
270 Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));
271 #else
272 if (munmap(Base.toPtr<void *>(), Size) != 0)
273 Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));
274 #endif
275
276 #elif defined(_WIN32)
277 (void)Size;
278
279 if (!UnmapViewOfFile(Base.toPtr<void *>()))
280 Err = joinErrors(std::move(Err),
281 errorCodeToError(mapWindowsError(GetLastError())));
282
283 CloseHandle(SharedMemoryFile);
284
285 #endif
286
287 std::lock_guard<std::mutex> Lock(Mutex);
288 Reservations.erase(Base.toPtr<void *>());
289 }
290
291 return Err;
292 #else
293 return make_error<StringError>(
294 "SharedMemoryMapper is not supported on this platform yet",
295 inconvertibleErrorCode());
296 #endif
297 }
298
shutdown()299 Error ExecutorSharedMemoryMapperService::shutdown() {
300 if (Reservations.empty())
301 return Error::success();
302
303 std::vector<ExecutorAddr> ReservationAddrs;
304 ReservationAddrs.reserve(Reservations.size());
305 for (const auto &R : Reservations)
306 ReservationAddrs.push_back(ExecutorAddr::fromPtr(R.getFirst()));
307
308 return release(std::move(ReservationAddrs));
309 }
310
addBootstrapSymbols(StringMap<ExecutorAddr> & M)311 void ExecutorSharedMemoryMapperService::addBootstrapSymbols(
312 StringMap<ExecutorAddr> &M) {
313 M[rt::ExecutorSharedMemoryMapperServiceInstanceName] =
314 ExecutorAddr::fromPtr(this);
315 M[rt::ExecutorSharedMemoryMapperServiceReserveWrapperName] =
316 ExecutorAddr::fromPtr(&reserveWrapper);
317 M[rt::ExecutorSharedMemoryMapperServiceInitializeWrapperName] =
318 ExecutorAddr::fromPtr(&initializeWrapper);
319 M[rt::ExecutorSharedMemoryMapperServiceDeinitializeWrapperName] =
320 ExecutorAddr::fromPtr(&deinitializeWrapper);
321 M[rt::ExecutorSharedMemoryMapperServiceReleaseWrapperName] =
322 ExecutorAddr::fromPtr(&releaseWrapper);
323 }
324
325 llvm::orc::shared::CWrapperFunctionResult
reserveWrapper(const char * ArgData,size_t ArgSize)326 ExecutorSharedMemoryMapperService::reserveWrapper(const char *ArgData,
327 size_t ArgSize) {
328 return shared::WrapperFunction<
329 rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>::
330 handle(ArgData, ArgSize,
331 shared::makeMethodWrapperHandler(
332 &ExecutorSharedMemoryMapperService::reserve))
333 .release();
334 }
335
336 llvm::orc::shared::CWrapperFunctionResult
initializeWrapper(const char * ArgData,size_t ArgSize)337 ExecutorSharedMemoryMapperService::initializeWrapper(const char *ArgData,
338 size_t ArgSize) {
339 return shared::WrapperFunction<
340 rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>::
341 handle(ArgData, ArgSize,
342 shared::makeMethodWrapperHandler(
343 &ExecutorSharedMemoryMapperService::initialize))
344 .release();
345 }
346
347 llvm::orc::shared::CWrapperFunctionResult
deinitializeWrapper(const char * ArgData,size_t ArgSize)348 ExecutorSharedMemoryMapperService::deinitializeWrapper(const char *ArgData,
349 size_t ArgSize) {
350 return shared::WrapperFunction<
351 rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>::
352 handle(ArgData, ArgSize,
353 shared::makeMethodWrapperHandler(
354 &ExecutorSharedMemoryMapperService::deinitialize))
355 .release();
356 }
357
358 llvm::orc::shared::CWrapperFunctionResult
releaseWrapper(const char * ArgData,size_t ArgSize)359 ExecutorSharedMemoryMapperService::releaseWrapper(const char *ArgData,
360 size_t ArgSize) {
361 return shared::WrapperFunction<
362 rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>::
363 handle(ArgData, ArgSize,
364 shared::makeMethodWrapperHandler(
365 &ExecutorSharedMemoryMapperService::release))
366 .release();
367 }
368
369 } // namespace rt_bootstrap
370 } // end namespace orc
371 } // end namespace llvm
372