|
83
|
1 //===- CompileOnDemandLayer.h - Compile each function on demand -*- C++ -*-===//
|
|
|
2 //
|
|
|
3 // The LLVM Compiler Infrastructure
|
|
|
4 //
|
|
|
5 // This file is distributed under the University of Illinois Open Source
|
|
|
6 // License. See LICENSE.TXT for details.
|
|
|
7 //
|
|
|
8 //===----------------------------------------------------------------------===//
|
|
|
9 //
|
|
|
10 // JIT layer for breaking up modules and inserting callbacks to allow
|
|
|
11 // individual functions to be compiled on demand.
|
|
|
12 //
|
|
|
13 //===----------------------------------------------------------------------===//
|
|
|
14
|
|
|
15 #ifndef LLVM_EXECUTIONENGINE_ORC_COMPILEONDEMANDLAYER_H
|
|
|
16 #define LLVM_EXECUTIONENGINE_ORC_COMPILEONDEMANDLAYER_H
|
|
|
17
|
|
120
|
18 #include "llvm/ADT/APInt.h"
|
|
83
|
19 #include "llvm/ADT/STLExtras.h"
|
|
120
|
20 #include "llvm/ADT/StringRef.h"
|
|
|
21 #include "llvm/ADT/Twine.h"
|
|
|
22 #include "llvm/ExecutionEngine/JITSymbol.h"
|
|
|
23 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
|
|
|
24 #include "llvm/ExecutionEngine/Orc/LambdaResolver.h"
|
|
121
|
25 #include "llvm/ExecutionEngine/Orc/OrcError.h"
|
|
|
26 #include "llvm/ExecutionEngine/RuntimeDyld.h"
|
|
120
|
27 #include "llvm/IR/Attributes.h"
|
|
121
|
28 #include "llvm/IR/Constant.h"
|
|
120
|
29 #include "llvm/IR/Constants.h"
|
|
|
30 #include "llvm/IR/DataLayout.h"
|
|
|
31 #include "llvm/IR/Function.h"
|
|
|
32 #include "llvm/IR/GlobalAlias.h"
|
|
|
33 #include "llvm/IR/GlobalValue.h"
|
|
|
34 #include "llvm/IR/GlobalVariable.h"
|
|
121
|
35 #include "llvm/IR/Instruction.h"
|
|
120
|
36 #include "llvm/IR/Mangler.h"
|
|
|
37 #include "llvm/IR/Module.h"
|
|
121
|
38 #include "llvm/IR/Type.h"
|
|
120
|
39 #include "llvm/Support/Casting.h"
|
|
|
40 #include "llvm/Support/raw_ostream.h"
|
|
121
|
41 #include "llvm/Transforms/Utils/ValueMapper.h"
|
|
120
|
42 #include <algorithm>
|
|
|
43 #include <cassert>
|
|
|
44 #include <functional>
|
|
|
45 #include <iterator>
|
|
83
|
46 #include <list>
|
|
100
|
47 #include <memory>
|
|
95
|
48 #include <set>
|
|
120
|
49 #include <string>
|
|
|
50 #include <utility>
|
|
|
51 #include <vector>
|
|
83
|
52
|
|
|
53 namespace llvm {
|
|
121
|
54
|
|
|
55 class Value;
|
|
|
56
|
|
95
|
57 namespace orc {
|
|
83
|
58
|
|
|
59 /// @brief Compile-on-demand layer.
|
|
|
60 ///
|
|
95
|
61 /// When a module is added to this layer a stub is created for each of its
|
|
|
62 /// function definitions. The stubs and other global values are immediately
|
|
|
63 /// added to the layer below. When a stub is called it triggers the extraction
|
|
|
64 /// of the function body from the original module. The extracted body is then
|
|
|
65 /// compiled and executed.
|
|
100
|
66 template <typename BaseLayerT,
|
|
|
67 typename CompileCallbackMgrT = JITCompileCallbackManager,
|
|
|
68 typename IndirectStubsMgrT = IndirectStubsManager>
|
|
83
|
69 class CompileOnDemandLayer {
|
|
95
|
70 private:
|
|
100
|
71 template <typename MaterializerFtor>
|
|
|
72 class LambdaMaterializer final : public ValueMaterializer {
|
|
83
|
73 public:
|
|
100
|
74 LambdaMaterializer(MaterializerFtor M) : M(std::move(M)) {}
|
|
120
|
75
|
|
|
76 Value *materialize(Value *V) final { return M(V); }
|
|
83
|
77
|
|
|
78 private:
|
|
100
|
79 MaterializerFtor M;
|
|
95
|
80 };
|
|
83
|
81
|
|
100
|
82 template <typename MaterializerFtor>
|
|
|
83 LambdaMaterializer<MaterializerFtor>
|
|
|
84 createLambdaMaterializer(MaterializerFtor M) {
|
|
|
85 return LambdaMaterializer<MaterializerFtor>(std::move(M));
|
|
|
86 }
|
|
|
87
|
|
121
|
88 using BaseLayerModuleHandleT = typename BaseLayerT::ModuleHandleT;
|
|
83
|
89
|
|
100
|
90 // Provide type-erasure for the Modules and MemoryManagers.
|
|
|
91 template <typename ResourceT>
|
|
|
92 class ResourceOwner {
|
|
|
93 public:
|
|
|
94 ResourceOwner() = default;
|
|
121
|
95 ResourceOwner(const ResourceOwner &) = delete;
|
|
|
96 ResourceOwner &operator=(const ResourceOwner &) = delete;
|
|
120
|
97 virtual ~ResourceOwner() = default;
|
|
|
98
|
|
100
|
99 virtual ResourceT& getResource() const = 0;
|
|
|
100 };
|
|
|
101
|
|
|
102 template <typename ResourceT, typename ResourcePtrT>
|
|
|
103 class ResourceOwnerImpl : public ResourceOwner<ResourceT> {
|
|
|
104 public:
|
|
|
105 ResourceOwnerImpl(ResourcePtrT ResourcePtr)
|
|
|
106 : ResourcePtr(std::move(ResourcePtr)) {}
|
|
120
|
107
|
|
100
|
108 ResourceT& getResource() const override { return *ResourcePtr; }
|
|
120
|
109
|
|
100
|
110 private:
|
|
|
111 ResourcePtrT ResourcePtr;
|
|
|
112 };
|
|
|
113
|
|
|
114 template <typename ResourceT, typename ResourcePtrT>
|
|
|
115 std::unique_ptr<ResourceOwner<ResourceT>>
|
|
|
116 wrapOwnership(ResourcePtrT ResourcePtr) {
|
|
121
|
117 using RO = ResourceOwnerImpl<ResourceT, ResourcePtrT>;
|
|
100
|
118 return llvm::make_unique<RO>(std::move(ResourcePtr));
|
|
|
119 }
|
|
|
120
|
|
120
|
121 class StaticGlobalRenamer {
|
|
|
122 public:
|
|
|
123 StaticGlobalRenamer() = default;
|
|
|
124 StaticGlobalRenamer(StaticGlobalRenamer &&) = default;
|
|
|
125 StaticGlobalRenamer &operator=(StaticGlobalRenamer &&) = default;
|
|
100
|
126
|
|
120
|
127 void rename(Module &M) {
|
|
|
128 for (auto &F : M)
|
|
|
129 if (F.hasLocalLinkage())
|
|
|
130 F.setName("$static." + Twine(NextId++));
|
|
|
131 for (auto &G : M.globals())
|
|
|
132 if (G.hasLocalLinkage())
|
|
|
133 G.setName("$static." + Twine(NextId++));
|
|
100
|
134 }
|
|
|
135
|
|
120
|
136 private:
|
|
|
137 unsigned NextId = 0;
|
|
83
|
138 };
|
|
|
139
|
|
120
|
140 struct LogicalDylib {
|
|
121
|
141 using SymbolResolverFtor = std::function<JITSymbol(const std::string&)>;
|
|
100
|
142
|
|
120
|
143 struct SourceModuleEntry {
|
|
121
|
144 std::shared_ptr<Module> SourceMod;
|
|
120
|
145 std::set<Function*> StubsToClone;
|
|
|
146 };
|
|
|
147
|
|
121
|
148 using SourceModulesList = std::vector<SourceModuleEntry>;
|
|
|
149 using SourceModuleHandle = typename SourceModulesList::size_type;
|
|
100
|
150
|
|
120
|
151 SourceModuleHandle
|
|
121
|
152 addSourceModule(std::shared_ptr<Module> M) {
|
|
120
|
153 SourceModuleHandle H = SourceModules.size();
|
|
|
154 SourceModules.push_back(SourceModuleEntry());
|
|
|
155 SourceModules.back().SourceMod = std::move(M);
|
|
|
156 return H;
|
|
|
157 }
|
|
100
|
158
|
|
120
|
159 Module& getSourceModule(SourceModuleHandle H) {
|
|
121
|
160 return *SourceModules[H].SourceMod;
|
|
100
|
161 }
|
|
|
162
|
|
120
|
163 std::set<Function*>& getStubsToClone(SourceModuleHandle H) {
|
|
|
164 return SourceModules[H].StubsToClone;
|
|
|
165 }
|
|
|
166
|
|
|
167 JITSymbol findSymbol(BaseLayerT &BaseLayer, const std::string &Name,
|
|
|
168 bool ExportedSymbolsOnly) {
|
|
|
169 if (auto Sym = StubsMgr->findStub(Name, ExportedSymbolsOnly))
|
|
|
170 return Sym;
|
|
|
171 for (auto BLH : BaseLayerHandles)
|
|
|
172 if (auto Sym = BaseLayer.findSymbolIn(BLH, Name, ExportedSymbolsOnly))
|
|
|
173 return Sym;
|
|
121
|
174 else if (auto Err = Sym.takeError())
|
|
|
175 return std::move(Err);
|
|
120
|
176 return nullptr;
|
|
|
177 }
|
|
|
178
|
|
121
|
179 Error removeModulesFromBaseLayer(BaseLayerT &BaseLayer) {
|
|
|
180 for (auto &BLH : BaseLayerHandles)
|
|
|
181 if (auto Err = BaseLayer.removeModule(BLH))
|
|
|
182 return Err;
|
|
|
183 return Error::success();
|
|
|
184 }
|
|
|
185
|
|
|
186 std::shared_ptr<JITSymbolResolver> ExternalSymbolResolver;
|
|
120
|
187 std::unique_ptr<IndirectStubsMgrT> StubsMgr;
|
|
|
188 StaticGlobalRenamer StaticRenamer;
|
|
|
189 SourceModulesList SourceModules;
|
|
121
|
190 std::vector<BaseLayerModuleHandleT> BaseLayerHandles;
|
|
83
|
191 };
|
|
|
192
|
|
121
|
193 using LogicalDylibList = std::list<LogicalDylib>;
|
|
83
|
194
|
|
|
195 public:
|
|
121
|
196
|
|
|
197 /// @brief Handle to loaded module.
|
|
|
198 using ModuleHandleT = typename LogicalDylibList::iterator;
|
|
83
|
199
|
|
100
|
200 /// @brief Module partitioning functor.
|
|
121
|
201 using PartitioningFtor = std::function<std::set<Function*>(Function&)>;
|
|
100
|
202
|
|
|
203 /// @brief Builder for IndirectStubsManagers.
|
|
121
|
204 using IndirectStubsManagerBuilderT =
|
|
|
205 std::function<std::unique_ptr<IndirectStubsMgrT>()>;
|
|
100
|
206
|
|
83
|
207 /// @brief Construct a compile-on-demand layer instance.
|
|
100
|
208 CompileOnDemandLayer(BaseLayerT &BaseLayer, PartitioningFtor Partition,
|
|
|
209 CompileCallbackMgrT &CallbackMgr,
|
|
|
210 IndirectStubsManagerBuilderT CreateIndirectStubsManager,
|
|
95
|
211 bool CloneStubsIntoPartitions = true)
|
|
120
|
212 : BaseLayer(BaseLayer), Partition(std::move(Partition)),
|
|
100
|
213 CompileCallbackMgr(CallbackMgr),
|
|
|
214 CreateIndirectStubsManager(std::move(CreateIndirectStubsManager)),
|
|
95
|
215 CloneStubsIntoPartitions(CloneStubsIntoPartitions) {}
|
|
83
|
216
|
|
121
|
217 ~CompileOnDemandLayer() {
|
|
|
218 // FIXME: Report error on log.
|
|
|
219 while (!LogicalDylibs.empty())
|
|
|
220 consumeError(removeModule(LogicalDylibs.begin()));
|
|
|
221 }
|
|
|
222
|
|
83
|
223 /// @brief Add a module to the compile-on-demand layer.
|
|
121
|
224 Expected<ModuleHandleT>
|
|
|
225 addModule(std::shared_ptr<Module> M,
|
|
|
226 std::shared_ptr<JITSymbolResolver> Resolver) {
|
|
83
|
227
|
|
120
|
228 LogicalDylibs.push_back(LogicalDylib());
|
|
|
229 auto &LD = LogicalDylibs.back();
|
|
|
230 LD.ExternalSymbolResolver = std::move(Resolver);
|
|
|
231 LD.StubsMgr = CreateIndirectStubsManager();
|
|
83
|
232
|
|
|
233 // Process each of the modules in this module set.
|
|
121
|
234 if (auto Err = addLogicalModule(LD, std::move(M)))
|
|
|
235 return std::move(Err);
|
|
83
|
236
|
|
95
|
237 return std::prev(LogicalDylibs.end());
|
|
83
|
238 }
|
|
|
239
|
|
121
|
240 /// @brief Add extra modules to an existing logical module.
|
|
|
241 Error addExtraModule(ModuleHandleT H, std::shared_ptr<Module> M) {
|
|
|
242 return addLogicalModule(*H, std::move(M));
|
|
|
243 }
|
|
|
244
|
|
83
|
245 /// @brief Remove the module represented by the given handle.
|
|
|
246 ///
|
|
|
247 /// This will remove all modules in the layers below that were derived from
|
|
|
248 /// the module represented by H.
|
|
121
|
249 Error removeModule(ModuleHandleT H) {
|
|
|
250 auto Err = H->removeModulesFromBaseLayer(BaseLayer);
|
|
95
|
251 LogicalDylibs.erase(H);
|
|
121
|
252 return Err;
|
|
83
|
253 }
|
|
|
254
|
|
|
255 /// @brief Search for the given named symbol.
|
|
|
256 /// @param Name The name of the symbol to search for.
|
|
|
257 /// @param ExportedSymbolsOnly If true, search only for exported symbols.
|
|
|
258 /// @return A handle for the given named symbol, if it exists.
|
|
|
259 JITSymbol findSymbol(StringRef Name, bool ExportedSymbolsOnly) {
|
|
100
|
260 for (auto LDI = LogicalDylibs.begin(), LDE = LogicalDylibs.end();
|
|
120
|
261 LDI != LDE; ++LDI) {
|
|
|
262 if (auto Sym = LDI->StubsMgr->findStub(Name, ExportedSymbolsOnly))
|
|
|
263 return Sym;
|
|
|
264 if (auto Sym = findSymbolIn(LDI, Name, ExportedSymbolsOnly))
|
|
|
265 return Sym;
|
|
121
|
266 else if (auto Err = Sym.takeError())
|
|
|
267 return std::move(Err);
|
|
120
|
268 }
|
|
83
|
269 return BaseLayer.findSymbol(Name, ExportedSymbolsOnly);
|
|
|
270 }
|
|
|
271
|
|
|
272 /// @brief Get the address of a symbol provided by this layer, or some layer
|
|
|
273 /// below this one.
|
|
121
|
274 JITSymbol findSymbolIn(ModuleHandleT H, const std::string &Name,
|
|
83
|
275 bool ExportedSymbolsOnly) {
|
|
120
|
276 return H->findSymbol(BaseLayer, Name, ExportedSymbolsOnly);
|
|
|
277 }
|
|
|
278
|
|
|
279 /// @brief Update the stub for the given function to point at FnBodyAddr.
|
|
|
280 /// This can be used to support re-optimization.
|
|
|
281 /// @return true if the function exists and the stub is updated, false
|
|
|
282 /// otherwise.
|
|
|
283 //
|
|
|
284 // FIXME: We should track and free associated resources (unused compile
|
|
|
285 // callbacks, uncompiled IR, and no-longer-needed/reachable function
|
|
|
286 // implementations).
|
|
121
|
287 Error updatePointer(std::string FuncName, JITTargetAddress FnBodyAddr) {
|
|
120
|
288 //Find out which logical dylib contains our symbol
|
|
|
289 auto LDI = LogicalDylibs.begin();
|
|
|
290 for (auto LDE = LogicalDylibs.end(); LDI != LDE; ++LDI) {
|
|
121
|
291 if (auto LMResources =
|
|
|
292 LDI->getLogicalModuleResourcesForSymbol(FuncName, false)) {
|
|
120
|
293 Module &SrcM = LMResources->SourceModule->getResource();
|
|
|
294 std::string CalledFnName = mangle(FuncName, SrcM.getDataLayout());
|
|
121
|
295 if (auto Err = LMResources->StubsMgr->updatePointer(CalledFnName,
|
|
|
296 FnBodyAddr))
|
|
|
297 return Err;
|
|
|
298 return Error::success();
|
|
120
|
299 }
|
|
|
300 }
|
|
121
|
301 return make_error<JITSymbolNotFound>(FuncName);
|
|
83
|
302 }
|
|
|
303
|
|
|
304 private:
|
|
121
|
305
|
|
|
306 Error addLogicalModule(LogicalDylib &LD, std::shared_ptr<Module> SrcMPtr) {
|
|
83
|
307
|
|
120
|
308 // Rename all static functions / globals to $static.X :
|
|
|
309 // This will unique the names across all modules in the logical dylib,
|
|
|
310 // simplifying symbol lookup.
|
|
|
311 LD.StaticRenamer.rename(*SrcMPtr);
|
|
83
|
312
|
|
95
|
313 // Bump the linkage and rename any anonymous/privote members in SrcM to
|
|
|
314 // ensure that everything will resolve properly after we partition SrcM.
|
|
100
|
315 makeAllSymbolsExternallyAccessible(*SrcMPtr);
|
|
95
|
316
|
|
|
317 // Create a logical module handle for SrcM within the logical dylib.
|
|
120
|
318 Module &SrcM = *SrcMPtr;
|
|
121
|
319 auto LMId = LD.addSourceModule(std::move(SrcMPtr));
|
|
83
|
320
|
|
120
|
321 // Create stub functions.
|
|
100
|
322 const DataLayout &DL = SrcM.getDataLayout();
|
|
|
323 {
|
|
|
324 typename IndirectStubsMgrT::StubInitsMap StubInits;
|
|
|
325 for (auto &F : SrcM) {
|
|
|
326 // Skip declarations.
|
|
|
327 if (F.isDeclaration())
|
|
|
328 continue;
|
|
83
|
329
|
|
120
|
330 // Skip weak functions for which we already have definitions.
|
|
|
331 auto MangledName = mangle(F.getName(), DL);
|
|
121
|
332 if (F.hasWeakLinkage() || F.hasLinkOnceLinkage()) {
|
|
120
|
333 if (auto Sym = LD.findSymbol(BaseLayer, MangledName, false))
|
|
|
334 continue;
|
|
121
|
335 else if (auto Err = Sym.takeError())
|
|
|
336 return std::move(Err);
|
|
|
337 }
|
|
120
|
338
|
|
100
|
339 // Record all functions defined by this module.
|
|
|
340 if (CloneStubsIntoPartitions)
|
|
120
|
341 LD.getStubsToClone(LMId).insert(&F);
|
|
100
|
342
|
|
|
343 // Create a callback, associate it with the stub for the function,
|
|
|
344 // and set the compile action to compile the partition containing the
|
|
|
345 // function.
|
|
121
|
346 if (auto CCInfoOrErr = CompileCallbackMgr.getCompileCallback()) {
|
|
|
347 auto &CCInfo = *CCInfoOrErr;
|
|
|
348 StubInits[MangledName] =
|
|
|
349 std::make_pair(CCInfo.getAddress(),
|
|
|
350 JITSymbolFlags::fromGlobalValue(F));
|
|
|
351 CCInfo.setCompileAction([this, &LD, LMId, &F]() -> JITTargetAddress {
|
|
|
352 if (auto FnImplAddrOrErr = this->extractAndCompile(LD, LMId, F))
|
|
|
353 return *FnImplAddrOrErr;
|
|
|
354 else {
|
|
|
355 // FIXME: Report error, return to 'abort' or something similar.
|
|
|
356 consumeError(FnImplAddrOrErr.takeError());
|
|
|
357 return 0;
|
|
|
358 }
|
|
|
359 });
|
|
|
360 } else
|
|
|
361 return CCInfoOrErr.takeError();
|
|
100
|
362 }
|
|
|
363
|
|
121
|
364 if (auto Err = LD.StubsMgr->createStubs(StubInits))
|
|
|
365 return Err;
|
|
83
|
366 }
|
|
|
367
|
|
120
|
368 // If this module doesn't contain any globals, aliases, or module flags then
|
|
|
369 // we can bail out early and avoid the overhead of creating and managing an
|
|
|
370 // empty globals module.
|
|
|
371 if (SrcM.global_empty() && SrcM.alias_empty() &&
|
|
|
372 !SrcM.getModuleFlagsMetadata())
|
|
121
|
373 return Error::success();
|
|
120
|
374
|
|
|
375 // Create the GlobalValues module.
|
|
|
376 auto GVsM = llvm::make_unique<Module>((SrcM.getName() + ".globals").str(),
|
|
|
377 SrcM.getContext());
|
|
|
378 GVsM->setDataLayout(DL);
|
|
|
379
|
|
|
380 ValueToValueMapTy VMap;
|
|
|
381
|
|
100
|
382 // Clone global variable decls.
|
|
|
383 for (auto &GV : SrcM.globals())
|
|
|
384 if (!GV.isDeclaration() && !VMap.count(&GV))
|
|
|
385 cloneGlobalVariableDecl(*GVsM, GV, &VMap);
|
|
83
|
386
|
|
95
|
387 // And the aliases.
|
|
100
|
388 for (auto &A : SrcM.aliases())
|
|
|
389 if (!VMap.count(&A))
|
|
|
390 cloneGlobalAliasDecl(*GVsM, A, VMap);
|
|
|
391
|
|
120
|
392 // Clone the module flags.
|
|
|
393 cloneModuleFlagsMetadata(*GVsM, SrcM, VMap);
|
|
|
394
|
|
100
|
395 // Now we need to clone the GV and alias initializers.
|
|
|
396
|
|
|
397 // Initializers may refer to functions declared (but not defined) in this
|
|
|
398 // module. Build a materializer to clone decls on demand.
|
|
121
|
399 Error MaterializerErrors = Error::success();
|
|
100
|
400 auto Materializer = createLambdaMaterializer(
|
|
121
|
401 [&LD, &GVsM, &MaterializerErrors](Value *V) -> Value* {
|
|
100
|
402 if (auto *F = dyn_cast<Function>(V)) {
|
|
|
403 // Decls in the original module just get cloned.
|
|
|
404 if (F->isDeclaration())
|
|
|
405 return cloneFunctionDecl(*GVsM, *F);
|
|
83
|
406
|
|
100
|
407 // Definitions in the original module (which we have emitted stubs
|
|
|
408 // for at this point) get turned into a constant alias to the stub
|
|
|
409 // instead.
|
|
|
410 const DataLayout &DL = GVsM->getDataLayout();
|
|
|
411 std::string FName = mangle(F->getName(), DL);
|
|
|
412 unsigned PtrBitWidth = DL.getPointerTypeSizeInBits(F->getType());
|
|
121
|
413 JITTargetAddress StubAddr = 0;
|
|
|
414
|
|
|
415 // Get the address for the stub. If we encounter an error while
|
|
|
416 // doing so, stash it in the MaterializerErrors variable and use a
|
|
|
417 // null address as a placeholder.
|
|
|
418 if (auto StubSym = LD.StubsMgr->findStub(FName, false)) {
|
|
|
419 if (auto StubAddrOrErr = StubSym.getAddress())
|
|
|
420 StubAddr = *StubAddrOrErr;
|
|
|
421 else
|
|
|
422 MaterializerErrors = joinErrors(std::move(MaterializerErrors),
|
|
|
423 StubAddrOrErr.takeError());
|
|
|
424 }
|
|
|
425
|
|
|
426 ConstantInt *StubAddrCI =
|
|
|
427 ConstantInt::get(GVsM->getContext(), APInt(PtrBitWidth, StubAddr));
|
|
100
|
428 Constant *Init = ConstantExpr::getCast(Instruction::IntToPtr,
|
|
121
|
429 StubAddrCI, F->getType());
|
|
100
|
430 return GlobalAlias::create(F->getFunctionType(),
|
|
|
431 F->getType()->getAddressSpace(),
|
|
|
432 F->getLinkage(), F->getName(),
|
|
|
433 Init, GVsM.get());
|
|
|
434 }
|
|
|
435 // else....
|
|
|
436 return nullptr;
|
|
|
437 });
|
|
|
438
|
|
|
439 // Clone the global variable initializers.
|
|
|
440 for (auto &GV : SrcM.globals())
|
|
95
|
441 if (!GV.isDeclaration())
|
|
100
|
442 moveGlobalVariableInitializer(GV, VMap, &Materializer);
|
|
83
|
443
|
|
100
|
444 // Clone the global alias initializers.
|
|
|
445 for (auto &A : SrcM.aliases()) {
|
|
|
446 auto *NewA = cast<GlobalAlias>(VMap[&A]);
|
|
|
447 assert(NewA && "Alias not cloned?");
|
|
|
448 Value *Init = MapValue(A.getAliasee(), VMap, RF_None, nullptr,
|
|
|
449 &Materializer);
|
|
|
450 NewA->setAliasee(cast<Constant>(Init));
|
|
|
451 }
|
|
|
452
|
|
121
|
453 if (MaterializerErrors)
|
|
|
454 return MaterializerErrors;
|
|
|
455
|
|
100
|
456 // Build a resolver for the globals module and add it to the base layer.
|
|
|
457 auto GVsResolver = createLambdaResolver(
|
|
121
|
458 [this, &LD](const std::string &Name) -> JITSymbol {
|
|
120
|
459 if (auto Sym = LD.StubsMgr->findStub(Name, false))
|
|
|
460 return Sym;
|
|
|
461 if (auto Sym = LD.findSymbol(BaseLayer, Name, false))
|
|
|
462 return Sym;
|
|
121
|
463 else if (auto Err = Sym.takeError())
|
|
|
464 return std::move(Err);
|
|
120
|
465 return LD.ExternalSymbolResolver->findSymbolInLogicalDylib(Name);
|
|
83
|
466 },
|
|
120
|
467 [&LD](const std::string &Name) {
|
|
|
468 return LD.ExternalSymbolResolver->findSymbol(Name);
|
|
83
|
469 });
|
|
|
470
|
|
121
|
471 if (auto GVsHOrErr =
|
|
|
472 BaseLayer.addModule(std::move(GVsM), std::move(GVsResolver)))
|
|
|
473 LD.BaseLayerHandles.push_back(*GVsHOrErr);
|
|
|
474 else
|
|
|
475 return GVsHOrErr.takeError();
|
|
|
476
|
|
|
477 return Error::success();
|
|
83
|
478 }
|
|
|
479
|
|
100
|
480 static std::string mangle(StringRef Name, const DataLayout &DL) {
|
|
83
|
481 std::string MangledName;
|
|
|
482 {
|
|
|
483 raw_string_ostream MangledNameStream(MangledName);
|
|
95
|
484 Mangler::getNameWithPrefix(MangledNameStream, Name, DL);
|
|
83
|
485 }
|
|
|
486 return MangledName;
|
|
|
487 }
|
|
|
488
|
|
121
|
489 Expected<JITTargetAddress>
|
|
120
|
490 extractAndCompile(LogicalDylib &LD,
|
|
|
491 typename LogicalDylib::SourceModuleHandle LMId,
|
|
|
492 Function &F) {
|
|
|
493 Module &SrcM = LD.getSourceModule(LMId);
|
|
95
|
494
|
|
|
495 // If F is a declaration we must already have compiled it.
|
|
|
496 if (F.isDeclaration())
|
|
|
497 return 0;
|
|
|
498
|
|
|
499 // Grab the name of the function being called here.
|
|
100
|
500 std::string CalledFnName = mangle(F.getName(), SrcM.getDataLayout());
|
|
95
|
501
|
|
121
|
502 JITTargetAddress CalledAddr = 0;
|
|
95
|
503 auto Part = Partition(F);
|
|
121
|
504 if (auto PartHOrErr = emitPartition(LD, LMId, Part)) {
|
|
|
505 auto &PartH = *PartHOrErr;
|
|
|
506 for (auto *SubF : Part) {
|
|
|
507 std::string FnName = mangle(SubF->getName(), SrcM.getDataLayout());
|
|
|
508 if (auto FnBodySym = BaseLayer.findSymbolIn(PartH, FnName, false)) {
|
|
|
509 if (auto FnBodyAddrOrErr = FnBodySym.getAddress()) {
|
|
|
510 JITTargetAddress FnBodyAddr = *FnBodyAddrOrErr;
|
|
95
|
511
|
|
121
|
512 // If this is the function we're calling record the address so we can
|
|
|
513 // return it from this function.
|
|
|
514 if (SubF == &F)
|
|
|
515 CalledAddr = FnBodyAddr;
|
|
95
|
516
|
|
121
|
517 // Update the function body pointer for the stub.
|
|
|
518 if (auto EC = LD.StubsMgr->updatePointer(FnName, FnBodyAddr))
|
|
|
519 return 0;
|
|
95
|
520
|
|
121
|
521 } else
|
|
|
522 return FnBodyAddrOrErr.takeError();
|
|
|
523 } else if (auto Err = FnBodySym.takeError())
|
|
|
524 return std::move(Err);
|
|
|
525 else
|
|
|
526 llvm_unreachable("Function not emitted for partition");
|
|
|
527 }
|
|
95
|
528
|
|
121
|
529 LD.BaseLayerHandles.push_back(PartH);
|
|
|
530 } else
|
|
|
531 return PartHOrErr.takeError();
|
|
95
|
532
|
|
|
533 return CalledAddr;
|
|
|
534 }
|
|
|
535
|
|
|
536 template <typename PartitionT>
|
|
121
|
537 Expected<BaseLayerModuleHandleT>
|
|
120
|
538 emitPartition(LogicalDylib &LD,
|
|
|
539 typename LogicalDylib::SourceModuleHandle LMId,
|
|
|
540 const PartitionT &Part) {
|
|
|
541 Module &SrcM = LD.getSourceModule(LMId);
|
|
95
|
542
|
|
|
543 // Create the module.
|
|
|
544 std::string NewName = SrcM.getName();
|
|
|
545 for (auto *F : Part) {
|
|
|
546 NewName += ".";
|
|
|
547 NewName += F->getName();
|
|
|
548 }
|
|
|
549
|
|
|
550 auto M = llvm::make_unique<Module>(NewName, SrcM.getContext());
|
|
|
551 M->setDataLayout(SrcM.getDataLayout());
|
|
|
552 ValueToValueMapTy VMap;
|
|
100
|
553
|
|
121
|
554 auto Materializer = createLambdaMaterializer([&LD, &LMId,
|
|
|
555 &M](Value *V) -> Value * {
|
|
100
|
556 if (auto *GV = dyn_cast<GlobalVariable>(V))
|
|
|
557 return cloneGlobalVariableDecl(*M, *GV);
|
|
|
558
|
|
|
559 if (auto *F = dyn_cast<Function>(V)) {
|
|
|
560 // Check whether we want to clone an available_externally definition.
|
|
120
|
561 if (!LD.getStubsToClone(LMId).count(F))
|
|
100
|
562 return cloneFunctionDecl(*M, *F);
|
|
|
563
|
|
|
564 // Ok - we want an inlinable stub. For that to work we need a decl
|
|
|
565 // for the stub pointer.
|
|
|
566 auto *StubPtr = createImplPointer(*F->getType(), *M,
|
|
|
567 F->getName() + "$stub_ptr", nullptr);
|
|
|
568 auto *ClonedF = cloneFunctionDecl(*M, *F);
|
|
|
569 makeStub(*ClonedF, *StubPtr);
|
|
|
570 ClonedF->setLinkage(GlobalValue::AvailableExternallyLinkage);
|
|
|
571 ClonedF->addFnAttr(Attribute::AlwaysInline);
|
|
|
572 return ClonedF;
|
|
|
573 }
|
|
|
574
|
|
|
575 if (auto *A = dyn_cast<GlobalAlias>(V)) {
|
|
|
576 auto *Ty = A->getValueType();
|
|
|
577 if (Ty->isFunctionTy())
|
|
|
578 return Function::Create(cast<FunctionType>(Ty),
|
|
|
579 GlobalValue::ExternalLinkage, A->getName(),
|
|
|
580 M.get());
|
|
|
581
|
|
|
582 return new GlobalVariable(*M, Ty, false, GlobalValue::ExternalLinkage,
|
|
|
583 nullptr, A->getName(), nullptr,
|
|
|
584 GlobalValue::NotThreadLocal,
|
|
|
585 A->getType()->getAddressSpace());
|
|
|
586 }
|
|
|
587
|
|
|
588 return nullptr;
|
|
|
589 });
|
|
95
|
590
|
|
|
591 // Create decls in the new module.
|
|
|
592 for (auto *F : Part)
|
|
|
593 cloneFunctionDecl(*M, *F, &VMap);
|
|
|
594
|
|
|
595 // Move the function bodies.
|
|
|
596 for (auto *F : Part)
|
|
100
|
597 moveFunctionBody(*F, VMap, &Materializer);
|
|
95
|
598
|
|
|
599 // Create memory manager and symbol resolver.
|
|
|
600 auto Resolver = createLambdaResolver(
|
|
121
|
601 [this, &LD](const std::string &Name) -> JITSymbol {
|
|
120
|
602 if (auto Sym = LD.findSymbol(BaseLayer, Name, false))
|
|
|
603 return Sym;
|
|
121
|
604 else if (auto Err = Sym.takeError())
|
|
|
605 return std::move(Err);
|
|
120
|
606 return LD.ExternalSymbolResolver->findSymbolInLogicalDylib(Name);
|
|
95
|
607 },
|
|
121
|
608 [&LD](const std::string &Name) {
|
|
120
|
609 return LD.ExternalSymbolResolver->findSymbol(Name);
|
|
95
|
610 });
|
|
100
|
611
|
|
121
|
612 return BaseLayer.addModule(std::move(M), std::move(Resolver));
|
|
95
|
613 }
|
|
|
614
|
|
83
|
615 BaseLayerT &BaseLayer;
|
|
100
|
616 PartitioningFtor Partition;
|
|
95
|
617 CompileCallbackMgrT &CompileCallbackMgr;
|
|
100
|
618 IndirectStubsManagerBuilderT CreateIndirectStubsManager;
|
|
|
619
|
|
95
|
620 LogicalDylibList LogicalDylibs;
|
|
|
621 bool CloneStubsIntoPartitions;
|
|
83
|
622 };
|
|
95
|
623
|
|
120
|
624 } // end namespace orc
|
|
121
|
625
|
|
120
|
626 } // end namespace llvm
|
|
83
|
627
|
|
|
628 #endif // LLVM_EXECUTIONENGINE_ORC_COMPILEONDEMANDLAYER_H
|
