121
|
1 //===- IndirectionUtils.h - Utilities for adding indirections ---*- C++ -*-===//
|
83
|
2 //
|
|
3 // The LLVM Compiler Infrastructure
|
|
4 //
|
|
5 // This file is distributed under the University of Illinois Open Source
|
|
6 // License. See LICENSE.TXT for details.
|
|
7 //
|
|
8 //===----------------------------------------------------------------------===//
|
|
9 //
|
|
10 // Contains utilities for adding indirections and breaking up modules.
|
|
11 //
|
|
12 //===----------------------------------------------------------------------===//
|
|
13
|
|
14 #ifndef LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
|
|
15 #define LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
|
|
16
|
120
|
17 #include "llvm/ADT/StringMap.h"
|
|
18 #include "llvm/ADT/StringRef.h"
|
|
19 #include "llvm/ADT/Twine.h"
|
|
20 #include "llvm/ExecutionEngine/JITSymbol.h"
|
|
21 #include "llvm/Support/Error.h"
|
|
22 #include "llvm/Support/Memory.h"
|
100
|
23 #include "llvm/Support/Process.h"
|
120
|
24 #include "llvm/Transforms/Utils/ValueMapper.h"
|
|
25 #include <algorithm>
|
|
26 #include <cassert>
|
|
27 #include <cstdint>
|
|
28 #include <functional>
|
|
29 #include <map>
|
|
30 #include <memory>
|
|
31 #include <system_error>
|
|
32 #include <utility>
|
|
33 #include <vector>
|
83
|
34
|
|
35 namespace llvm {
|
121
|
36
|
|
37 class Constant;
|
|
38 class Function;
|
|
39 class FunctionType;
|
|
40 class GlobalAlias;
|
|
41 class GlobalVariable;
|
|
42 class Module;
|
|
43 class PointerType;
|
|
44 class Triple;
|
|
45 class Value;
|
|
46
|
95
|
47 namespace orc {
|
83
|
48
|
100
|
49 /// @brief Target-independent base class for compile callback management.
|
|
50 class JITCompileCallbackManager {
|
83
|
51 public:
|
121
|
52 using CompileFtor = std::function<JITTargetAddress()>;
|
95
|
53
|
|
54 /// @brief Handle to a newly created compile callback. Can be used to get an
|
|
55 /// IR constant representing the address of the trampoline, and to set
|
|
56 /// the compile action for the callback.
|
|
57 class CompileCallbackInfo {
|
|
58 public:
|
120
|
59 CompileCallbackInfo(JITTargetAddress Addr, CompileFtor &Compile)
|
|
60 : Addr(Addr), Compile(Compile) {}
|
95
|
61
|
120
|
62 JITTargetAddress getAddress() const { return Addr; }
|
95
|
63 void setCompileAction(CompileFtor Compile) {
|
|
64 this->Compile = std::move(Compile);
|
|
65 }
|
120
|
66
|
95
|
67 private:
|
120
|
68 JITTargetAddress Addr;
|
95
|
69 CompileFtor &Compile;
|
|
70 };
|
|
71
|
100
|
72 /// @brief Construct a JITCompileCallbackManager.
|
83
|
73 /// @param ErrorHandlerAddress The address of an error handler in the target
|
|
74 /// process to be used if a compile callback fails.
|
120
|
75 JITCompileCallbackManager(JITTargetAddress ErrorHandlerAddress)
|
|
76 : ErrorHandlerAddress(ErrorHandlerAddress) {}
|
83
|
77
|
120
|
78 virtual ~JITCompileCallbackManager() = default;
|
95
|
79
|
83
|
80 /// @brief Execute the callback for the given trampoline id. Called by the JIT
|
|
81 /// to compile functions on demand.
|
120
|
82 JITTargetAddress executeCompileCallback(JITTargetAddress TrampolineAddr) {
|
95
|
83 auto I = ActiveTrampolines.find(TrampolineAddr);
|
83
|
84 // FIXME: Also raise an error in the Orc error-handler when we finally have
|
|
85 // one.
|
|
86 if (I == ActiveTrampolines.end())
|
|
87 return ErrorHandlerAddress;
|
|
88
|
|
89 // Found a callback handler. Yank this trampoline out of the active list and
|
|
90 // put it back in the available trampolines list, then try to run the
|
|
91 // handler's compile and update actions.
|
120
|
92 // Moving the trampoline ID back to the available list first means there's
|
|
93 // at
|
|
94 // least one available trampoline if the compile action triggers a request
|
|
95 // for
|
83
|
96 // a new one.
|
95
|
97 auto Compile = std::move(I->second);
|
83
|
98 ActiveTrampolines.erase(I);
|
95
|
99 AvailableTrampolines.push_back(TrampolineAddr);
|
|
100
|
|
101 if (auto Addr = Compile())
|
|
102 return Addr;
|
|
103
|
|
104 return ErrorHandlerAddress;
|
|
105 }
|
|
106
|
|
107 /// @brief Reserve a compile callback.
|
121
|
108 Expected<CompileCallbackInfo> getCompileCallback() {
|
|
109 if (auto TrampolineAddrOrErr = getAvailableTrampolineAddr()) {
|
|
110 const auto &TrampolineAddr = *TrampolineAddrOrErr;
|
|
111 auto &Compile = this->ActiveTrampolines[TrampolineAddr];
|
|
112 return CompileCallbackInfo(TrampolineAddr, Compile);
|
|
113 } else
|
|
114 return TrampolineAddrOrErr.takeError();
|
100
|
115 }
|
83
|
116
|
95
|
117 /// @brief Get a CompileCallbackInfo for an existing callback.
|
120
|
118 CompileCallbackInfo getCompileCallbackInfo(JITTargetAddress TrampolineAddr) {
|
95
|
119 auto I = ActiveTrampolines.find(TrampolineAddr);
|
|
120 assert(I != ActiveTrampolines.end() && "Not an active trampoline.");
|
|
121 return CompileCallbackInfo(I->first, I->second);
|
|
122 }
|
|
123
|
|
124 /// @brief Release a compile callback.
|
|
125 ///
|
|
126 /// Note: Callbacks are auto-released after they execute. This method should
|
|
127 /// only be called to manually release a callback that is not going to
|
|
128 /// execute.
|
120
|
129 void releaseCompileCallback(JITTargetAddress TrampolineAddr) {
|
95
|
130 auto I = ActiveTrampolines.find(TrampolineAddr);
|
|
131 assert(I != ActiveTrampolines.end() && "Not an active trampoline.");
|
|
132 ActiveTrampolines.erase(I);
|
|
133 AvailableTrampolines.push_back(TrampolineAddr);
|
83
|
134 }
|
|
135
|
|
136 protected:
|
120
|
137 JITTargetAddress ErrorHandlerAddress;
|
83
|
138
|
121
|
139 using TrampolineMapT = std::map<JITTargetAddress, CompileFtor>;
|
83
|
140 TrampolineMapT ActiveTrampolines;
|
120
|
141 std::vector<JITTargetAddress> AvailableTrampolines;
|
83
|
142
|
|
143 private:
|
121
|
144 Expected<JITTargetAddress> getAvailableTrampolineAddr() {
|
83
|
145 if (this->AvailableTrampolines.empty())
|
121
|
146 if (auto Err = grow())
|
|
147 return std::move(Err);
|
83
|
148 assert(!this->AvailableTrampolines.empty() &&
|
|
149 "Failed to grow available trampolines.");
|
120
|
150 JITTargetAddress TrampolineAddr = this->AvailableTrampolines.back();
|
83
|
151 this->AvailableTrampolines.pop_back();
|
|
152 return TrampolineAddr;
|
|
153 }
|
|
154
|
100
|
155 // Create new trampolines - to be implemented in subclasses.
|
121
|
156 virtual Error grow() = 0;
|
100
|
157
|
|
158 virtual void anchor();
|
|
159 };
|
|
160
|
|
161 /// @brief Manage compile callbacks for in-process JITs.
|
|
162 template <typename TargetT>
|
|
163 class LocalJITCompileCallbackManager : public JITCompileCallbackManager {
|
|
164 public:
|
|
165 /// @brief Construct a InProcessJITCompileCallbackManager.
|
|
166 /// @param ErrorHandlerAddress The address of an error handler in the target
|
|
167 /// process to be used if a compile callback fails.
|
120
|
168 LocalJITCompileCallbackManager(JITTargetAddress ErrorHandlerAddress)
|
|
169 : JITCompileCallbackManager(ErrorHandlerAddress) {
|
100
|
170 /// Set up the resolver block.
|
|
171 std::error_code EC;
|
120
|
172 ResolverBlock = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(
|
|
173 TargetT::ResolverCodeSize, nullptr,
|
|
174 sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));
|
100
|
175 assert(!EC && "Failed to allocate resolver block");
|
|
176
|
|
177 TargetT::writeResolverCode(static_cast<uint8_t *>(ResolverBlock.base()),
|
|
178 &reenter, this);
|
|
179
|
|
180 EC = sys::Memory::protectMappedMemory(ResolverBlock.getMemoryBlock(),
|
|
181 sys::Memory::MF_READ |
|
|
182 sys::Memory::MF_EXEC);
|
|
183 assert(!EC && "Failed to mprotect resolver block");
|
|
184 }
|
|
185
|
|
186 private:
|
120
|
187 static JITTargetAddress reenter(void *CCMgr, void *TrampolineId) {
|
100
|
188 JITCompileCallbackManager *Mgr =
|
120
|
189 static_cast<JITCompileCallbackManager *>(CCMgr);
|
100
|
190 return Mgr->executeCompileCallback(
|
120
|
191 static_cast<JITTargetAddress>(
|
|
192 reinterpret_cast<uintptr_t>(TrampolineId)));
|
100
|
193 }
|
|
194
|
121
|
195 Error grow() override {
|
83
|
196 assert(this->AvailableTrampolines.empty() && "Growing prematurely?");
|
100
|
197
|
|
198 std::error_code EC;
|
|
199 auto TrampolineBlock =
|
120
|
200 sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(
|
|
201 sys::Process::getPageSize(), nullptr,
|
|
202 sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));
|
121
|
203 if (EC)
|
|
204 return errorCodeToError(EC);
|
100
|
205
|
|
206 unsigned NumTrampolines =
|
120
|
207 (sys::Process::getPageSize() - TargetT::PointerSize) /
|
100
|
208 TargetT::TrampolineSize;
|
|
209
|
120
|
210 uint8_t *TrampolineMem = static_cast<uint8_t *>(TrampolineBlock.base());
|
100
|
211 TargetT::writeTrampolines(TrampolineMem, ResolverBlock.base(),
|
|
212 NumTrampolines);
|
|
213
|
|
214 for (unsigned I = 0; I < NumTrampolines; ++I)
|
|
215 this->AvailableTrampolines.push_back(
|
120
|
216 static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(
|
100
|
217 TrampolineMem + (I * TargetT::TrampolineSize))));
|
|
218
|
121
|
219 if (auto EC = sys::Memory::protectMappedMemory(
|
|
220 TrampolineBlock.getMemoryBlock(),
|
|
221 sys::Memory::MF_READ | sys::Memory::MF_EXEC))
|
|
222 return errorCodeToError(EC);
|
100
|
223
|
|
224 TrampolineBlocks.push_back(std::move(TrampolineBlock));
|
121
|
225 return Error::success();
|
83
|
226 }
|
|
227
|
100
|
228 sys::OwningMemoryBlock ResolverBlock;
|
|
229 std::vector<sys::OwningMemoryBlock> TrampolineBlocks;
|
|
230 };
|
|
231
|
|
232 /// @brief Base class for managing collections of named indirect stubs.
|
|
233 class IndirectStubsManager {
|
|
234 public:
|
120
|
235 /// @brief Map type for initializing the manager. See init.
|
121
|
236 using StubInitsMap = StringMap<std::pair<JITTargetAddress, JITSymbolFlags>>;
|
100
|
237
|
120
|
238 virtual ~IndirectStubsManager() = default;
|
100
|
239
|
|
240 /// @brief Create a single stub with the given name, target address and flags.
|
120
|
241 virtual Error createStub(StringRef StubName, JITTargetAddress StubAddr,
|
|
242 JITSymbolFlags StubFlags) = 0;
|
100
|
243
|
|
244 /// @brief Create StubInits.size() stubs with the given names, target
|
|
245 /// addresses, and flags.
|
120
|
246 virtual Error createStubs(const StubInitsMap &StubInits) = 0;
|
100
|
247
|
|
248 /// @brief Find the stub with the given name. If ExportedStubsOnly is true,
|
|
249 /// this will only return a result if the stub's flags indicate that it
|
|
250 /// is exported.
|
|
251 virtual JITSymbol findStub(StringRef Name, bool ExportedStubsOnly) = 0;
|
|
252
|
|
253 /// @brief Find the implementation-pointer for the stub.
|
|
254 virtual JITSymbol findPointer(StringRef Name) = 0;
|
|
255
|
|
256 /// @brief Change the value of the implementation pointer for the stub.
|
120
|
257 virtual Error updatePointer(StringRef Name, JITTargetAddress NewAddr) = 0;
|
|
258
|
100
|
259 private:
|
|
260 virtual void anchor();
|
|
261 };
|
|
262
|
|
263 /// @brief IndirectStubsManager implementation for the host architecture, e.g.
|
|
264 /// OrcX86_64. (See OrcArchitectureSupport.h).
|
|
265 template <typename TargetT>
|
|
266 class LocalIndirectStubsManager : public IndirectStubsManager {
|
|
267 public:
|
120
|
268 Error createStub(StringRef StubName, JITTargetAddress StubAddr,
|
|
269 JITSymbolFlags StubFlags) override {
|
|
270 if (auto Err = reserveStubs(1))
|
|
271 return Err;
|
100
|
272
|
|
273 createStubInternal(StubName, StubAddr, StubFlags);
|
|
274
|
120
|
275 return Error::success();
|
100
|
276 }
|
|
277
|
120
|
278 Error createStubs(const StubInitsMap &StubInits) override {
|
|
279 if (auto Err = reserveStubs(StubInits.size()))
|
|
280 return Err;
|
100
|
281
|
|
282 for (auto &Entry : StubInits)
|
|
283 createStubInternal(Entry.first(), Entry.second.first,
|
|
284 Entry.second.second);
|
|
285
|
120
|
286 return Error::success();
|
100
|
287 }
|
|
288
|
|
289 JITSymbol findStub(StringRef Name, bool ExportedStubsOnly) override {
|
|
290 auto I = StubIndexes.find(Name);
|
|
291 if (I == StubIndexes.end())
|
|
292 return nullptr;
|
|
293 auto Key = I->second.first;
|
|
294 void *StubAddr = IndirectStubsInfos[Key.first].getStub(Key.second);
|
|
295 assert(StubAddr && "Missing stub address");
|
|
296 auto StubTargetAddr =
|
120
|
297 static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(StubAddr));
|
100
|
298 auto StubSymbol = JITSymbol(StubTargetAddr, I->second.second);
|
120
|
299 if (ExportedStubsOnly && !StubSymbol.getFlags().isExported())
|
100
|
300 return nullptr;
|
|
301 return StubSymbol;
|
|
302 }
|
|
303
|
|
304 JITSymbol findPointer(StringRef Name) override {
|
|
305 auto I = StubIndexes.find(Name);
|
|
306 if (I == StubIndexes.end())
|
|
307 return nullptr;
|
|
308 auto Key = I->second.first;
|
|
309 void *PtrAddr = IndirectStubsInfos[Key.first].getPtr(Key.second);
|
|
310 assert(PtrAddr && "Missing pointer address");
|
|
311 auto PtrTargetAddr =
|
120
|
312 static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(PtrAddr));
|
100
|
313 return JITSymbol(PtrTargetAddr, I->second.second);
|
|
314 }
|
|
315
|
120
|
316 Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override {
|
100
|
317 auto I = StubIndexes.find(Name);
|
|
318 assert(I != StubIndexes.end() && "No stub pointer for symbol");
|
|
319 auto Key = I->second.first;
|
|
320 *IndirectStubsInfos[Key.first].getPtr(Key.second) =
|
120
|
321 reinterpret_cast<void *>(static_cast<uintptr_t>(NewAddr));
|
|
322 return Error::success();
|
100
|
323 }
|
|
324
|
|
325 private:
|
120
|
326 Error reserveStubs(unsigned NumStubs) {
|
100
|
327 if (NumStubs <= FreeStubs.size())
|
120
|
328 return Error::success();
|
100
|
329
|
|
330 unsigned NewStubsRequired = NumStubs - FreeStubs.size();
|
|
331 unsigned NewBlockId = IndirectStubsInfos.size();
|
|
332 typename TargetT::IndirectStubsInfo ISI;
|
120
|
333 if (auto Err =
|
|
334 TargetT::emitIndirectStubsBlock(ISI, NewStubsRequired, nullptr))
|
|
335 return Err;
|
100
|
336 for (unsigned I = 0; I < ISI.getNumStubs(); ++I)
|
|
337 FreeStubs.push_back(std::make_pair(NewBlockId, I));
|
|
338 IndirectStubsInfos.push_back(std::move(ISI));
|
120
|
339 return Error::success();
|
100
|
340 }
|
|
341
|
120
|
342 void createStubInternal(StringRef StubName, JITTargetAddress InitAddr,
|
100
|
343 JITSymbolFlags StubFlags) {
|
|
344 auto Key = FreeStubs.back();
|
|
345 FreeStubs.pop_back();
|
|
346 *IndirectStubsInfos[Key.first].getPtr(Key.second) =
|
120
|
347 reinterpret_cast<void *>(static_cast<uintptr_t>(InitAddr));
|
100
|
348 StubIndexes[StubName] = std::make_pair(Key, StubFlags);
|
|
349 }
|
|
350
|
|
351 std::vector<typename TargetT::IndirectStubsInfo> IndirectStubsInfos;
|
121
|
352 using StubKey = std::pair<uint16_t, uint16_t>;
|
100
|
353 std::vector<StubKey> FreeStubs;
|
|
354 StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes;
|
83
|
355 };
|
|
356
|
120
|
357 /// @brief Create a local compile callback manager.
|
|
358 ///
|
|
359 /// The given target triple will determine the ABI, and the given
|
|
360 /// ErrorHandlerAddress will be used by the resulting compile callback
|
|
361 /// manager if a compile callback fails.
|
|
362 std::unique_ptr<JITCompileCallbackManager>
|
|
363 createLocalCompileCallbackManager(const Triple &T,
|
|
364 JITTargetAddress ErrorHandlerAddress);
|
|
365
|
|
366 /// @brief Create a local indriect stubs manager builder.
|
|
367 ///
|
|
368 /// The given target triple will determine the ABI.
|
|
369 std::function<std::unique_ptr<IndirectStubsManager>()>
|
|
370 createLocalIndirectStubsManagerBuilder(const Triple &T);
|
|
371
|
95
|
372 /// @brief Build a function pointer of FunctionType with the given constant
|
|
373 /// address.
|
|
374 ///
|
|
375 /// Usage example: Turn a trampoline address into a function pointer constant
|
|
376 /// for use in a stub.
|
120
|
377 Constant *createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr);
|
83
|
378
|
95
|
379 /// @brief Create a function pointer with the given type, name, and initializer
|
|
380 /// in the given Module.
|
120
|
381 GlobalVariable *createImplPointer(PointerType &PT, Module &M, const Twine &Name,
|
|
382 Constant *Initializer);
|
95
|
383
|
|
384 /// @brief Turn a function declaration into a stub function that makes an
|
|
385 /// indirect call using the given function pointer.
|
100
|
386 void makeStub(Function &F, Value &ImplPointer);
|
83
|
387
|
95
|
388 /// @brief Raise linkage types and rename as necessary to ensure that all
|
|
389 /// symbols are accessible for other modules.
|
|
390 ///
|
|
391 /// This should be called before partitioning a module to ensure that the
|
|
392 /// partitions retain access to each other's symbols.
|
|
393 void makeAllSymbolsExternallyAccessible(Module &M);
|
83
|
394
|
95
|
395 /// @brief Clone a function declaration into a new module.
|
|
396 ///
|
|
397 /// This function can be used as the first step towards creating a callback
|
|
398 /// stub (see makeStub), or moving a function body (see moveFunctionBody).
|
|
399 ///
|
|
400 /// If the VMap argument is non-null, a mapping will be added between F and
|
|
401 /// the new declaration, and between each of F's arguments and the new
|
|
402 /// declaration's arguments. This map can then be passed in to moveFunction to
|
|
403 /// move the function body if required. Note: When moving functions between
|
|
404 /// modules with these utilities, all decls should be cloned (and added to a
|
|
405 /// single VMap) before any bodies are moved. This will ensure that references
|
|
406 /// between functions all refer to the versions in the new module.
|
120
|
407 Function *cloneFunctionDecl(Module &Dst, const Function &F,
|
95
|
408 ValueToValueMapTy *VMap = nullptr);
|
83
|
409
|
95
|
410 /// @brief Move the body of function 'F' to a cloned function declaration in a
|
|
411 /// different module (See related cloneFunctionDecl).
|
|
412 ///
|
|
413 /// If the target function declaration is not supplied via the NewF parameter
|
|
414 /// then it will be looked up via the VMap.
|
|
415 ///
|
|
416 /// This will delete the body of function 'F' from its original parent module,
|
|
417 /// but leave its declaration.
|
|
418 void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
|
|
419 ValueMaterializer *Materializer = nullptr,
|
|
420 Function *NewF = nullptr);
|
|
421
|
|
422 /// @brief Clone a global variable declaration into a new module.
|
120
|
423 GlobalVariable *cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
|
95
|
424 ValueToValueMapTy *VMap = nullptr);
|
83
|
425
|
95
|
426 /// @brief Move global variable GV from its parent module to cloned global
|
|
427 /// declaration in a different module.
|
|
428 ///
|
|
429 /// If the target global declaration is not supplied via the NewGV parameter
|
|
430 /// then it will be looked up via the VMap.
|
|
431 ///
|
|
432 /// This will delete the initializer of GV from its original parent module,
|
|
433 /// but leave its declaration.
|
|
434 void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
|
|
435 ValueToValueMapTy &VMap,
|
|
436 ValueMaterializer *Materializer = nullptr,
|
|
437 GlobalVariable *NewGV = nullptr);
|
83
|
438
|
120
|
439 /// @brief Clone a global alias declaration into a new module.
|
|
440 GlobalAlias *cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
|
100
|
441 ValueToValueMapTy &VMap);
|
83
|
442
|
120
|
443 /// @brief Clone module flags metadata into the destination module.
|
|
444 void cloneModuleFlagsMetadata(Module &Dst, const Module &Src,
|
|
445 ValueToValueMapTy &VMap);
|
|
446
|
|
447 } // end namespace orc
|
121
|
448
|
120
|
449 } // end namespace llvm
|
83
|
450
|
|
451 #endif // LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
|