|
120
|
1 //===-ThinLTOCodeGenerator.cpp - LLVM Link Time Optimizer -----------------===//
|
|
|
2 //
|
|
147
|
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
4 // See https://llvm.org/LICENSE.txt for license information.
|
|
|
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
120
|
6 //
|
|
|
7 //===----------------------------------------------------------------------===//
|
|
|
8 //
|
|
|
9 // This file implements the Thin Link Time Optimization library. This library is
|
|
|
10 // intended to be used by linker to optimize code at link time.
|
|
|
11 //
|
|
|
12 //===----------------------------------------------------------------------===//
|
|
|
13
|
|
|
14 #include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
|
|
|
15
|
|
|
16 #include "llvm/ADT/Statistic.h"
|
|
|
17 #include "llvm/ADT/StringExtras.h"
|
|
|
18 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
|
|
|
19 #include "llvm/Analysis/ProfileSummaryInfo.h"
|
|
|
20 #include "llvm/Analysis/TargetLibraryInfo.h"
|
|
|
21 #include "llvm/Analysis/TargetTransformInfo.h"
|
|
|
22 #include "llvm/Bitcode/BitcodeReader.h"
|
|
|
23 #include "llvm/Bitcode/BitcodeWriter.h"
|
|
|
24 #include "llvm/Bitcode/BitcodeWriterPass.h"
|
|
147
|
25 #include "llvm/Config/llvm-config.h"
|
|
121
|
26 #include "llvm/IR/DebugInfo.h"
|
|
120
|
27 #include "llvm/IR/DiagnosticPrinter.h"
|
|
|
28 #include "llvm/IR/LLVMContext.h"
|
|
|
29 #include "llvm/IR/LegacyPassManager.h"
|
|
|
30 #include "llvm/IR/Mangler.h"
|
|
147
|
31 #include "llvm/IR/PassTimingInfo.h"
|
|
|
32 #include "llvm/IR/RemarkStreamer.h"
|
|
121
|
33 #include "llvm/IR/Verifier.h"
|
|
120
|
34 #include "llvm/IRReader/IRReader.h"
|
|
|
35 #include "llvm/LTO/LTO.h"
|
|
147
|
36 #include "llvm/LTO/SummaryBasedOptimizations.h"
|
|
120
|
37 #include "llvm/MC/SubtargetFeature.h"
|
|
|
38 #include "llvm/Object/IRObjectFile.h"
|
|
|
39 #include "llvm/Support/CachePruning.h"
|
|
|
40 #include "llvm/Support/Debug.h"
|
|
|
41 #include "llvm/Support/Error.h"
|
|
|
42 #include "llvm/Support/Path.h"
|
|
|
43 #include "llvm/Support/SHA1.h"
|
|
147
|
44 #include "llvm/Support/SmallVectorMemoryBuffer.h"
|
|
120
|
45 #include "llvm/Support/TargetRegistry.h"
|
|
|
46 #include "llvm/Support/ThreadPool.h"
|
|
|
47 #include "llvm/Support/Threading.h"
|
|
|
48 #include "llvm/Support/ToolOutputFile.h"
|
|
121
|
49 #include "llvm/Support/VCSRevision.h"
|
|
120
|
50 #include "llvm/Target/TargetMachine.h"
|
|
|
51 #include "llvm/Transforms/IPO.h"
|
|
|
52 #include "llvm/Transforms/IPO/FunctionImport.h"
|
|
|
53 #include "llvm/Transforms/IPO/Internalize.h"
|
|
|
54 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
|
|
|
55 #include "llvm/Transforms/ObjCARC.h"
|
|
|
56 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
|
|
|
57
|
|
|
58 #include <numeric>
|
|
|
59
|
|
147
|
60 #if !defined(_MSC_VER) && !defined(__MINGW32__)
|
|
|
61 #include <unistd.h>
|
|
|
62 #else
|
|
|
63 #include <io.h>
|
|
|
64 #endif
|
|
|
65
|
|
120
|
66 using namespace llvm;
|
|
|
67
|
|
|
68 #define DEBUG_TYPE "thinlto"
|
|
|
69
|
|
|
70 namespace llvm {
|
|
|
71 // Flags -discard-value-names, defined in LTOCodeGenerator.cpp
|
|
|
72 extern cl::opt<bool> LTODiscardValueNames;
|
|
147
|
73 extern cl::opt<std::string> RemarksFilename;
|
|
|
74 extern cl::opt<std::string> RemarksPasses;
|
|
|
75 extern cl::opt<bool> RemarksWithHotness;
|
|
|
76 extern cl::opt<std::string> RemarksFormat;
|
|
120
|
77 }
|
|
|
78
|
|
|
79 namespace {
|
|
|
80
|
|
|
81 static cl::opt<int>
|
|
|
82 ThreadCount("threads", cl::init(llvm::heavyweight_hardware_concurrency()));
|
|
|
83
|
|
|
84 // Simple helper to save temporary files for debug.
|
|
|
85 static void saveTempBitcode(const Module &TheModule, StringRef TempDir,
|
|
|
86 unsigned count, StringRef Suffix) {
|
|
|
87 if (TempDir.empty())
|
|
|
88 return;
|
|
|
89 // User asked to save temps, let dump the bitcode file after import.
|
|
134
|
90 std::string SaveTempPath = (TempDir + llvm::Twine(count) + Suffix).str();
|
|
120
|
91 std::error_code EC;
|
|
147
|
92 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
|
|
120
|
93 if (EC)
|
|
|
94 report_fatal_error(Twine("Failed to open ") + SaveTempPath +
|
|
|
95 " to save optimized bitcode\n");
|
|
134
|
96 WriteBitcodeToFile(TheModule, OS, /* ShouldPreserveUseListOrder */ true);
|
|
120
|
97 }
|
|
|
98
|
|
|
99 static const GlobalValueSummary *
|
|
|
100 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
|
|
|
101 // If there is any strong definition anywhere, get it.
|
|
|
102 auto StrongDefForLinker = llvm::find_if(
|
|
|
103 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
|
|
|
104 auto Linkage = Summary->linkage();
|
|
|
105 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
|
|
|
106 !GlobalValue::isWeakForLinker(Linkage);
|
|
|
107 });
|
|
|
108 if (StrongDefForLinker != GVSummaryList.end())
|
|
|
109 return StrongDefForLinker->get();
|
|
|
110 // Get the first *linker visible* definition for this global in the summary
|
|
|
111 // list.
|
|
|
112 auto FirstDefForLinker = llvm::find_if(
|
|
|
113 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
|
|
|
114 auto Linkage = Summary->linkage();
|
|
|
115 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
|
|
|
116 });
|
|
|
117 // Extern templates can be emitted as available_externally.
|
|
|
118 if (FirstDefForLinker == GVSummaryList.end())
|
|
|
119 return nullptr;
|
|
|
120 return FirstDefForLinker->get();
|
|
|
121 }
|
|
|
122
|
|
|
123 // Populate map of GUID to the prevailing copy for any multiply defined
|
|
|
124 // symbols. Currently assume first copy is prevailing, or any strong
|
|
|
125 // definition. Can be refined with Linker information in the future.
|
|
|
126 static void computePrevailingCopies(
|
|
|
127 const ModuleSummaryIndex &Index,
|
|
|
128 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy) {
|
|
|
129 auto HasMultipleCopies = [&](const GlobalValueSummaryList &GVSummaryList) {
|
|
|
130 return GVSummaryList.size() > 1;
|
|
|
131 };
|
|
|
132
|
|
|
133 for (auto &I : Index) {
|
|
121
|
134 if (HasMultipleCopies(I.second.SummaryList))
|
|
|
135 PrevailingCopy[I.first] =
|
|
|
136 getFirstDefinitionForLinker(I.second.SummaryList);
|
|
120
|
137 }
|
|
|
138 }
|
|
|
139
|
|
147
|
140 static StringMap<lto::InputFile *>
|
|
|
141 generateModuleMap(std::vector<std::unique_ptr<lto::InputFile>> &Modules) {
|
|
|
142 StringMap<lto::InputFile *> ModuleMap;
|
|
|
143 for (auto &M : Modules) {
|
|
|
144 assert(ModuleMap.find(M->getName()) == ModuleMap.end() &&
|
|
120
|
145 "Expect unique Buffer Identifier");
|
|
147
|
146 ModuleMap[M->getName()] = M.get();
|
|
120
|
147 }
|
|
|
148 return ModuleMap;
|
|
|
149 }
|
|
|
150
|
|
|
151 static void promoteModule(Module &TheModule, const ModuleSummaryIndex &Index) {
|
|
|
152 if (renameModuleForThinLTO(TheModule, Index))
|
|
|
153 report_fatal_error("renameModuleForThinLTO failed");
|
|
|
154 }
|
|
|
155
|
|
121
|
156 namespace {
|
|
|
157 class ThinLTODiagnosticInfo : public DiagnosticInfo {
|
|
|
158 const Twine &Msg;
|
|
|
159 public:
|
|
|
160 ThinLTODiagnosticInfo(const Twine &DiagMsg,
|
|
|
161 DiagnosticSeverity Severity = DS_Error)
|
|
|
162 : DiagnosticInfo(DK_Linker, Severity), Msg(DiagMsg) {}
|
|
|
163 void print(DiagnosticPrinter &DP) const override { DP << Msg; }
|
|
|
164 };
|
|
|
165 }
|
|
|
166
|
|
|
167 /// Verify the module and strip broken debug info.
|
|
|
168 static void verifyLoadedModule(Module &TheModule) {
|
|
|
169 bool BrokenDebugInfo = false;
|
|
|
170 if (verifyModule(TheModule, &dbgs(), &BrokenDebugInfo))
|
|
|
171 report_fatal_error("Broken module found, compilation aborted!");
|
|
|
172 if (BrokenDebugInfo) {
|
|
|
173 TheModule.getContext().diagnose(ThinLTODiagnosticInfo(
|
|
|
174 "Invalid debug info found, debug info will be stripped", DS_Warning));
|
|
|
175 StripDebugInfo(TheModule);
|
|
|
176 }
|
|
|
177 }
|
|
|
178
|
|
147
|
179 static std::unique_ptr<Module> loadModuleFromInput(lto::InputFile *Input,
|
|
|
180 LLVMContext &Context,
|
|
|
181 bool Lazy,
|
|
|
182 bool IsImporting) {
|
|
|
183 auto &Mod = Input->getSingleBitcodeModule();
|
|
121
|
184 SMDiagnostic Err;
|
|
|
185 Expected<std::unique_ptr<Module>> ModuleOrErr =
|
|
147
|
186 Lazy ? Mod.getLazyModule(Context,
|
|
|
187 /* ShouldLazyLoadMetadata */ true, IsImporting)
|
|
|
188 : Mod.parseModule(Context);
|
|
121
|
189 if (!ModuleOrErr) {
|
|
|
190 handleAllErrors(ModuleOrErr.takeError(), [&](ErrorInfoBase &EIB) {
|
|
147
|
191 SMDiagnostic Err = SMDiagnostic(Mod.getModuleIdentifier(),
|
|
121
|
192 SourceMgr::DK_Error, EIB.message());
|
|
|
193 Err.print("ThinLTO", errs());
|
|
|
194 });
|
|
|
195 report_fatal_error("Can't load module, abort.");
|
|
|
196 }
|
|
|
197 if (!Lazy)
|
|
|
198 verifyLoadedModule(*ModuleOrErr.get());
|
|
147
|
199 return std::move(*ModuleOrErr);
|
|
121
|
200 }
|
|
|
201
|
|
120
|
202 static void
|
|
|
203 crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index,
|
|
147
|
204 StringMap<lto::InputFile*> &ModuleMap,
|
|
120
|
205 const FunctionImporter::ImportMapTy &ImportList) {
|
|
121
|
206 auto Loader = [&](StringRef Identifier) {
|
|
147
|
207 auto &Input = ModuleMap[Identifier];
|
|
|
208 return loadModuleFromInput(Input, TheModule.getContext(),
|
|
|
209 /*Lazy=*/true, /*IsImporting*/ true);
|
|
121
|
210 };
|
|
|
211
|
|
120
|
212 FunctionImporter Importer(Index, Loader);
|
|
121
|
213 Expected<bool> Result = Importer.importFunctions(TheModule, ImportList);
|
|
|
214 if (!Result) {
|
|
|
215 handleAllErrors(Result.takeError(), [&](ErrorInfoBase &EIB) {
|
|
|
216 SMDiagnostic Err = SMDiagnostic(TheModule.getModuleIdentifier(),
|
|
|
217 SourceMgr::DK_Error, EIB.message());
|
|
|
218 Err.print("ThinLTO", errs());
|
|
|
219 });
|
|
120
|
220 report_fatal_error("importFunctions failed");
|
|
121
|
221 }
|
|
|
222 // Verify again after cross-importing.
|
|
|
223 verifyLoadedModule(TheModule);
|
|
120
|
224 }
|
|
|
225
|
|
121
|
226 static void optimizeModule(Module &TheModule, TargetMachine &TM,
|
|
|
227 unsigned OptLevel, bool Freestanding) {
|
|
120
|
228 // Populate the PassManager
|
|
|
229 PassManagerBuilder PMB;
|
|
|
230 PMB.LibraryInfo = new TargetLibraryInfoImpl(TM.getTargetTriple());
|
|
121
|
231 if (Freestanding)
|
|
|
232 PMB.LibraryInfo->disableAllFunctions();
|
|
120
|
233 PMB.Inliner = createFunctionInliningPass();
|
|
|
234 // FIXME: should get it from the bitcode?
|
|
121
|
235 PMB.OptLevel = OptLevel;
|
|
120
|
236 PMB.LoopVectorize = true;
|
|
|
237 PMB.SLPVectorize = true;
|
|
121
|
238 // Already did this in verifyLoadedModule().
|
|
|
239 PMB.VerifyInput = false;
|
|
120
|
240 PMB.VerifyOutput = false;
|
|
|
241
|
|
|
242 legacy::PassManager PM;
|
|
|
243
|
|
|
244 // Add the TTI (required to inform the vectorizer about register size for
|
|
|
245 // instance)
|
|
|
246 PM.add(createTargetTransformInfoWrapperPass(TM.getTargetIRAnalysis()));
|
|
|
247
|
|
|
248 // Add optimizations
|
|
|
249 PMB.populateThinLTOPassManager(PM);
|
|
|
250
|
|
|
251 PM.run(TheModule);
|
|
|
252 }
|
|
|
253
|
|
147
|
254 static void
|
|
|
255 addUsedSymbolToPreservedGUID(const lto::InputFile &File,
|
|
|
256 DenseSet<GlobalValue::GUID> &PreservedGUID) {
|
|
|
257 for (const auto &Sym : File.symbols()) {
|
|
|
258 if (Sym.isUsed())
|
|
|
259 PreservedGUID.insert(GlobalValue::getGUID(Sym.getIRName()));
|
|
|
260 }
|
|
|
261 }
|
|
|
262
|
|
120
|
263 // Convert the PreservedSymbols map from "Name" based to "GUID" based.
|
|
|
264 static DenseSet<GlobalValue::GUID>
|
|
|
265 computeGUIDPreservedSymbols(const StringSet<> &PreservedSymbols,
|
|
|
266 const Triple &TheTriple) {
|
|
|
267 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size());
|
|
|
268 for (auto &Entry : PreservedSymbols) {
|
|
|
269 StringRef Name = Entry.first();
|
|
|
270 if (TheTriple.isOSBinFormatMachO() && Name.size() > 0 && Name[0] == '_')
|
|
|
271 Name = Name.drop_front();
|
|
|
272 GUIDPreservedSymbols.insert(GlobalValue::getGUID(Name));
|
|
|
273 }
|
|
|
274 return GUIDPreservedSymbols;
|
|
|
275 }
|
|
|
276
|
|
|
277 std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
|
|
|
278 TargetMachine &TM) {
|
|
|
279 SmallVector<char, 128> OutputBuffer;
|
|
|
280
|
|
|
281 // CodeGen
|
|
|
282 {
|
|
|
283 raw_svector_ostream OS(OutputBuffer);
|
|
|
284 legacy::PassManager PM;
|
|
|
285
|
|
|
286 // If the bitcode files contain ARC code and were compiled with optimization,
|
|
|
287 // the ObjCARCContractPass must be run, so do it unconditionally here.
|
|
|
288 PM.add(createObjCARCContractPass());
|
|
|
289
|
|
|
290 // Setup the codegen now.
|
|
147
|
291 if (TM.addPassesToEmitFile(PM, OS, nullptr, TargetMachine::CGFT_ObjectFile,
|
|
120
|
292 /* DisableVerify */ true))
|
|
|
293 report_fatal_error("Failed to setup codegen");
|
|
|
294
|
|
|
295 // Run codegen now. resulting binary is in OutputBuffer.
|
|
|
296 PM.run(TheModule);
|
|
|
297 }
|
|
147
|
298 return make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer));
|
|
120
|
299 }
|
|
|
300
|
|
|
301 /// Manage caching for a single Module.
|
|
|
302 class ModuleCacheEntry {
|
|
|
303 SmallString<128> EntryPath;
|
|
|
304
|
|
|
305 public:
|
|
|
306 // Create a cache entry. This compute a unique hash for the Module considering
|
|
|
307 // the current list of export/import, and offer an interface to query to
|
|
|
308 // access the content in the cache.
|
|
|
309 ModuleCacheEntry(
|
|
|
310 StringRef CachePath, const ModuleSummaryIndex &Index, StringRef ModuleID,
|
|
|
311 const FunctionImporter::ImportMapTy &ImportList,
|
|
|
312 const FunctionImporter::ExportSetTy &ExportList,
|
|
|
313 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
|
|
147
|
314 const GVSummaryMapTy &DefinedGVSummaries, unsigned OptLevel,
|
|
121
|
315 bool Freestanding, const TargetMachineBuilder &TMBuilder) {
|
|
120
|
316 if (CachePath.empty())
|
|
|
317 return;
|
|
|
318
|
|
|
319 if (!Index.modulePaths().count(ModuleID))
|
|
|
320 // The module does not have an entry, it can't have a hash at all
|
|
|
321 return;
|
|
|
322
|
|
147
|
323 if (all_of(Index.getModuleHash(ModuleID),
|
|
|
324 [](uint32_t V) { return V == 0; }))
|
|
120
|
325 // No hash entry, no caching!
|
|
|
326 return;
|
|
|
327
|
|
147
|
328 llvm::lto::Config Conf;
|
|
|
329 Conf.OptLevel = OptLevel;
|
|
|
330 Conf.Options = TMBuilder.Options;
|
|
|
331 Conf.CPU = TMBuilder.MCpu;
|
|
|
332 Conf.MAttrs.push_back(TMBuilder.MAttr);
|
|
|
333 Conf.RelocModel = TMBuilder.RelocModel;
|
|
|
334 Conf.CGOptLevel = TMBuilder.CGOptLevel;
|
|
|
335 Conf.Freestanding = Freestanding;
|
|
|
336 SmallString<40> Key;
|
|
|
337 computeLTOCacheKey(Key, Conf, Index, ModuleID, ImportList, ExportList,
|
|
|
338 ResolvedODR, DefinedGVSummaries);
|
|
120
|
339
|
|
121
|
340 // This choice of file name allows the cache to be pruned (see pruneCache()
|
|
|
341 // in include/llvm/Support/CachePruning.h).
|
|
147
|
342 sys::path::append(EntryPath, CachePath, "llvmcache-" + Key);
|
|
120
|
343 }
|
|
|
344
|
|
|
345 // Access the path to this entry in the cache.
|
|
|
346 StringRef getEntryPath() { return EntryPath; }
|
|
|
347
|
|
|
348 // Try loading the buffer for this cache entry.
|
|
|
349 ErrorOr<std::unique_ptr<MemoryBuffer>> tryLoadingBuffer() {
|
|
|
350 if (EntryPath.empty())
|
|
|
351 return std::error_code();
|
|
147
|
352 SmallString<64> ResultPath;
|
|
|
353 Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead(
|
|
|
354 Twine(EntryPath), sys::fs::OF_UpdateAtime, &ResultPath);
|
|
|
355 if (!FDOrErr)
|
|
|
356 return errorToErrorCode(FDOrErr.takeError());
|
|
|
357 ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = MemoryBuffer::getOpenFile(
|
|
|
358 *FDOrErr, EntryPath, /*FileSize=*/-1, /*RequiresNullTerminator=*/false);
|
|
|
359 sys::fs::closeFile(*FDOrErr);
|
|
|
360 return MBOrErr;
|
|
120
|
361 }
|
|
|
362
|
|
|
363 // Cache the Produced object file
|
|
121
|
364 void write(const MemoryBuffer &OutputBuffer) {
|
|
120
|
365 if (EntryPath.empty())
|
|
121
|
366 return;
|
|
120
|
367
|
|
|
368 // Write to a temporary to avoid race condition
|
|
|
369 SmallString<128> TempFilename;
|
|
147
|
370 SmallString<128> CachePath(EntryPath);
|
|
120
|
371 int TempFD;
|
|
147
|
372 llvm::sys::path::remove_filename(CachePath);
|
|
|
373 sys::path::append(TempFilename, CachePath, "Thin-%%%%%%.tmp.o");
|
|
120
|
374 std::error_code EC =
|
|
147
|
375 sys::fs::createUniqueFile(TempFilename, TempFD, TempFilename);
|
|
120
|
376 if (EC) {
|
|
|
377 errs() << "Error: " << EC.message() << "\n";
|
|
|
378 report_fatal_error("ThinLTO: Can't get a temporary file");
|
|
|
379 }
|
|
|
380 {
|
|
|
381 raw_fd_ostream OS(TempFD, /* ShouldClose */ true);
|
|
121
|
382 OS << OutputBuffer.getBuffer();
|
|
120
|
383 }
|
|
147
|
384 // Rename temp file to final destination; rename is atomic
|
|
120
|
385 EC = sys::fs::rename(TempFilename, EntryPath);
|
|
147
|
386 if (EC)
|
|
120
|
387 sys::fs::remove(TempFilename);
|
|
|
388 }
|
|
|
389 };
|
|
|
390
|
|
|
391 static std::unique_ptr<MemoryBuffer>
|
|
|
392 ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
|
|
147
|
393 StringMap<lto::InputFile *> &ModuleMap, TargetMachine &TM,
|
|
120
|
394 const FunctionImporter::ImportMapTy &ImportList,
|
|
|
395 const FunctionImporter::ExportSetTy &ExportList,
|
|
|
396 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
|
|
|
397 const GVSummaryMapTy &DefinedGlobals,
|
|
|
398 const ThinLTOCodeGenerator::CachingOptions &CacheOptions,
|
|
|
399 bool DisableCodeGen, StringRef SaveTempsDir,
|
|
121
|
400 bool Freestanding, unsigned OptLevel, unsigned count) {
|
|
120
|
401
|
|
|
402 // "Benchmark"-like optimization: single-source case
|
|
|
403 bool SingleModule = (ModuleMap.size() == 1);
|
|
|
404
|
|
|
405 if (!SingleModule) {
|
|
|
406 promoteModule(TheModule, Index);
|
|
|
407
|
|
147
|
408 // Apply summary-based prevailing-symbol resolution decisions.
|
|
|
409 thinLTOResolvePrevailingInModule(TheModule, DefinedGlobals);
|
|
120
|
410
|
|
|
411 // Save temps: after promotion.
|
|
|
412 saveTempBitcode(TheModule, SaveTempsDir, count, ".1.promoted.bc");
|
|
|
413 }
|
|
|
414
|
|
|
415 // Be friendly and don't nuke totally the module when the client didn't
|
|
|
416 // supply anything to preserve.
|
|
|
417 if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) {
|
|
|
418 // Apply summary-based internalization decisions.
|
|
|
419 thinLTOInternalizeModule(TheModule, DefinedGlobals);
|
|
|
420 }
|
|
|
421
|
|
|
422 // Save internalized bitcode
|
|
|
423 saveTempBitcode(TheModule, SaveTempsDir, count, ".2.internalized.bc");
|
|
|
424
|
|
|
425 if (!SingleModule) {
|
|
|
426 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList);
|
|
|
427
|
|
|
428 // Save temps: after cross-module import.
|
|
|
429 saveTempBitcode(TheModule, SaveTempsDir, count, ".3.imported.bc");
|
|
|
430 }
|
|
|
431
|
|
121
|
432 optimizeModule(TheModule, TM, OptLevel, Freestanding);
|
|
120
|
433
|
|
|
434 saveTempBitcode(TheModule, SaveTempsDir, count, ".4.opt.bc");
|
|
|
435
|
|
|
436 if (DisableCodeGen) {
|
|
|
437 // Configured to stop before CodeGen, serialize the bitcode and return.
|
|
|
438 SmallVector<char, 128> OutputBuffer;
|
|
|
439 {
|
|
|
440 raw_svector_ostream OS(OutputBuffer);
|
|
|
441 ProfileSummaryInfo PSI(TheModule);
|
|
121
|
442 auto Index = buildModuleSummaryIndex(TheModule, nullptr, &PSI);
|
|
134
|
443 WriteBitcodeToFile(TheModule, OS, true, &Index);
|
|
120
|
444 }
|
|
147
|
445 return make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer));
|
|
120
|
446 }
|
|
|
447
|
|
|
448 return codegenModule(TheModule, TM);
|
|
|
449 }
|
|
|
450
|
|
147
|
451 /// Resolve prevailing symbols. Record resolutions in the \p ResolvedODR map
|
|
120
|
452 /// for caching, and in the \p Index for application during the ThinLTO
|
|
|
453 /// backends. This is needed for correctness for exported symbols (ensure
|
|
|
454 /// at least one copy kept) and a compile-time optimization (to drop duplicate
|
|
|
455 /// copies when possible).
|
|
147
|
456 static void resolvePrevailingInIndex(
|
|
120
|
457 ModuleSummaryIndex &Index,
|
|
|
458 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>>
|
|
147
|
459 &ResolvedODR,
|
|
|
460 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
|
|
120
|
461
|
|
|
462 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
|
|
|
463 computePrevailingCopies(Index, PrevailingCopy);
|
|
|
464
|
|
|
465 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
|
|
|
466 const auto &Prevailing = PrevailingCopy.find(GUID);
|
|
|
467 // Not in map means that there was only one copy, which must be prevailing.
|
|
|
468 if (Prevailing == PrevailingCopy.end())
|
|
|
469 return true;
|
|
|
470 return Prevailing->second == S;
|
|
|
471 };
|
|
|
472
|
|
|
473 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
|
|
|
474 GlobalValue::GUID GUID,
|
|
|
475 GlobalValue::LinkageTypes NewLinkage) {
|
|
|
476 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
|
|
|
477 };
|
|
|
478
|
|
147
|
479 thinLTOResolvePrevailingInIndex(Index, isPrevailing, recordNewLinkage,
|
|
|
480 GUIDPreservedSymbols);
|
|
120
|
481 }
|
|
|
482
|
|
|
483 // Initialize the TargetMachine builder for a given Triple
|
|
|
484 static void initTMBuilder(TargetMachineBuilder &TMBuilder,
|
|
|
485 const Triple &TheTriple) {
|
|
|
486 // Set a default CPU for Darwin triples (copied from LTOCodeGenerator).
|
|
|
487 // FIXME this looks pretty terrible...
|
|
|
488 if (TMBuilder.MCpu.empty() && TheTriple.isOSDarwin()) {
|
|
|
489 if (TheTriple.getArch() == llvm::Triple::x86_64)
|
|
|
490 TMBuilder.MCpu = "core2";
|
|
|
491 else if (TheTriple.getArch() == llvm::Triple::x86)
|
|
|
492 TMBuilder.MCpu = "yonah";
|
|
|
493 else if (TheTriple.getArch() == llvm::Triple::aarch64)
|
|
|
494 TMBuilder.MCpu = "cyclone";
|
|
|
495 }
|
|
|
496 TMBuilder.TheTriple = std::move(TheTriple);
|
|
|
497 }
|
|
|
498
|
|
|
499 } // end anonymous namespace
|
|
|
500
|
|
|
501 void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
|
|
147
|
502 MemoryBufferRef Buffer(Data, Identifier);
|
|
121
|
503
|
|
147
|
504 auto InputOrError = lto::InputFile::create(Buffer);
|
|
|
505 if (!InputOrError)
|
|
|
506 report_fatal_error("ThinLTO cannot create input file: " +
|
|
|
507 toString(InputOrError.takeError()));
|
|
121
|
508
|
|
147
|
509 auto TripleStr = (*InputOrError)->getTargetTriple();
|
|
121
|
510 Triple TheTriple(TripleStr);
|
|
|
511
|
|
|
512 if (Modules.empty())
|
|
120
|
513 initTMBuilder(TMBuilder, Triple(TheTriple));
|
|
121
|
514 else if (TMBuilder.TheTriple != TheTriple) {
|
|
|
515 if (!TMBuilder.TheTriple.isCompatibleWith(TheTriple))
|
|
|
516 report_fatal_error("ThinLTO modules with incompatible triples not "
|
|
|
517 "supported");
|
|
|
518 initTMBuilder(TMBuilder, Triple(TMBuilder.TheTriple.merge(TheTriple)));
|
|
120
|
519 }
|
|
121
|
520
|
|
147
|
521 Modules.emplace_back(std::move(*InputOrError));
|
|
120
|
522 }
|
|
|
523
|
|
|
524 void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) {
|
|
|
525 PreservedSymbols.insert(Name);
|
|
|
526 }
|
|
|
527
|
|
|
528 void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) {
|
|
|
529 // FIXME: At the moment, we don't take advantage of this extra information,
|
|
|
530 // we're conservatively considering cross-references as preserved.
|
|
|
531 // CrossReferencedSymbols.insert(Name);
|
|
|
532 PreservedSymbols.insert(Name);
|
|
|
533 }
|
|
|
534
|
|
|
535 // TargetMachine factory
|
|
|
536 std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const {
|
|
|
537 std::string ErrMsg;
|
|
|
538 const Target *TheTarget =
|
|
|
539 TargetRegistry::lookupTarget(TheTriple.str(), ErrMsg);
|
|
|
540 if (!TheTarget) {
|
|
|
541 report_fatal_error("Can't load target for this Triple: " + ErrMsg);
|
|
|
542 }
|
|
|
543
|
|
|
544 // Use MAttr as the default set of features.
|
|
|
545 SubtargetFeatures Features(MAttr);
|
|
|
546 Features.getDefaultSubtargetFeatures(TheTriple);
|
|
|
547 std::string FeatureStr = Features.getString();
|
|
121
|
548
|
|
|
549 return std::unique_ptr<TargetMachine>(
|
|
|
550 TheTarget->createTargetMachine(TheTriple.str(), MCpu, FeatureStr, Options,
|
|
|
551 RelocModel, None, CGOptLevel));
|
|
120
|
552 }
|
|
|
553
|
|
|
554 /**
|
|
|
555 * Produce the combined summary index from all the bitcode files:
|
|
|
556 * "thin-link".
|
|
|
557 */
|
|
|
558 std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
|
|
121
|
559 std::unique_ptr<ModuleSummaryIndex> CombinedIndex =
|
|
147
|
560 llvm::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/false);
|
|
120
|
561 uint64_t NextModuleId = 0;
|
|
147
|
562 for (auto &Mod : Modules) {
|
|
|
563 auto &M = Mod->getSingleBitcodeModule();
|
|
|
564 if (Error Err =
|
|
|
565 M.readSummary(*CombinedIndex, Mod->getName(), NextModuleId++)) {
|
|
120
|
566 // FIXME diagnose
|
|
|
567 logAllUnhandledErrors(
|
|
121
|
568 std::move(Err), errs(),
|
|
|
569 "error: can't create module summary index for buffer: ");
|
|
120
|
570 return nullptr;
|
|
|
571 }
|
|
|
572 }
|
|
|
573 return CombinedIndex;
|
|
|
574 }
|
|
|
575
|
|
134
|
576 static void internalizeAndPromoteInIndex(
|
|
|
577 const StringMap<FunctionImporter::ExportSetTy> &ExportLists,
|
|
|
578 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
|
|
|
579 ModuleSummaryIndex &Index) {
|
|
|
580 auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
|
|
|
581 const auto &ExportList = ExportLists.find(ModuleIdentifier);
|
|
|
582 return (ExportList != ExportLists.end() &&
|
|
|
583 ExportList->second.count(GUID)) ||
|
|
|
584 GUIDPreservedSymbols.count(GUID);
|
|
|
585 };
|
|
|
586
|
|
|
587 thinLTOInternalizeAndPromoteInIndex(Index, isExported);
|
|
|
588 }
|
|
|
589
|
|
|
590 static void computeDeadSymbolsInIndex(
|
|
|
591 ModuleSummaryIndex &Index,
|
|
|
592 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
|
|
|
593 // We have no symbols resolution available. And can't do any better now in the
|
|
|
594 // case where the prevailing symbol is in a native object. It can be refined
|
|
|
595 // with linker information in the future.
|
|
|
596 auto isPrevailing = [&](GlobalValue::GUID G) {
|
|
|
597 return PrevailingType::Unknown;
|
|
|
598 };
|
|
147
|
599 computeDeadSymbolsWithConstProp(Index, GUIDPreservedSymbols, isPrevailing,
|
|
|
600 /* ImportEnabled = */ true);
|
|
134
|
601 }
|
|
|
602
|
|
120
|
603 /**
|
|
|
604 * Perform promotion and renaming of exported internal functions.
|
|
|
605 * Index is updated to reflect linkage changes from weak resolution.
|
|
|
606 */
|
|
147
|
607 void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index,
|
|
|
608 const lto::InputFile &File) {
|
|
120
|
609 auto ModuleCount = Index.modulePaths().size();
|
|
|
610 auto ModuleIdentifier = TheModule.getModuleIdentifier();
|
|
|
611
|
|
|
612 // Collect for each module the list of function it defines (GUID -> Summary).
|
|
|
613 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
|
|
|
614 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
|
|
|
615
|
|
121
|
616 // Convert the preserved symbols set from string to GUID
|
|
|
617 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
|
|
|
618 PreservedSymbols, Triple(TheModule.getTargetTriple()));
|
|
|
619
|
|
147
|
620 // Add used symbol to the preserved symbols.
|
|
|
621 addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
|
|
|
622
|
|
121
|
623 // Compute "dead" symbols, we don't want to import/export these!
|
|
134
|
624 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
|
|
121
|
625
|
|
120
|
626 // Generate import/export list
|
|
|
627 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
|
|
|
628 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
|
|
|
629 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
|
|
|
630 ExportLists);
|
|
|
631
|
|
147
|
632 // Resolve prevailing symbols
|
|
120
|
633 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
|
|
147
|
634 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols);
|
|
120
|
635
|
|
147
|
636 thinLTOResolvePrevailingInModule(
|
|
120
|
637 TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]);
|
|
|
638
|
|
|
639 // Promote the exported values in the index, so that they are promoted
|
|
|
640 // in the module.
|
|
134
|
641 internalizeAndPromoteInIndex(ExportLists, GUIDPreservedSymbols, Index);
|
|
120
|
642
|
|
|
643 promoteModule(TheModule, Index);
|
|
|
644 }
|
|
|
645
|
|
|
646 /**
|
|
|
647 * Perform cross-module importing for the module identified by ModuleIdentifier.
|
|
|
648 */
|
|
|
649 void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
|
|
147
|
650 ModuleSummaryIndex &Index,
|
|
|
651 const lto::InputFile &File) {
|
|
120
|
652 auto ModuleMap = generateModuleMap(Modules);
|
|
|
653 auto ModuleCount = Index.modulePaths().size();
|
|
|
654
|
|
|
655 // Collect for each module the list of function it defines (GUID -> Summary).
|
|
|
656 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
|
|
|
657 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
|
|
|
658
|
|
121
|
659 // Convert the preserved symbols set from string to GUID
|
|
|
660 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
|
|
|
661 PreservedSymbols, Triple(TheModule.getTargetTriple()));
|
|
|
662
|
|
147
|
663 addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
|
|
|
664
|
|
121
|
665 // Compute "dead" symbols, we don't want to import/export these!
|
|
134
|
666 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
|
|
121
|
667
|
|
120
|
668 // Generate import/export list
|
|
|
669 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
|
|
|
670 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
|
|
|
671 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
|
|
|
672 ExportLists);
|
|
|
673 auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
|
|
|
674
|
|
|
675 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList);
|
|
|
676 }
|
|
|
677
|
|
|
678 /**
|
|
|
679 * Compute the list of summaries needed for importing into module.
|
|
|
680 */
|
|
|
681 void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
|
|
147
|
682 Module &TheModule, ModuleSummaryIndex &Index,
|
|
|
683 std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex,
|
|
|
684 const lto::InputFile &File) {
|
|
120
|
685 auto ModuleCount = Index.modulePaths().size();
|
|
147
|
686 auto ModuleIdentifier = TheModule.getModuleIdentifier();
|
|
120
|
687
|
|
|
688 // Collect for each module the list of function it defines (GUID -> Summary).
|
|
|
689 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
|
|
|
690 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
|
|
|
691
|
|
147
|
692 // Convert the preserved symbols set from string to GUID
|
|
|
693 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
|
|
|
694 PreservedSymbols, Triple(TheModule.getTargetTriple()));
|
|
|
695
|
|
|
696 addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
|
|
|
697
|
|
|
698 // Compute "dead" symbols, we don't want to import/export these!
|
|
|
699 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
|
|
|
700
|
|
120
|
701 // Generate import/export list
|
|
|
702 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
|
|
|
703 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
|
|
|
704 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
|
|
|
705 ExportLists);
|
|
|
706
|
|
147
|
707 llvm::gatherImportedSummariesForModule(
|
|
|
708 ModuleIdentifier, ModuleToDefinedGVSummaries,
|
|
|
709 ImportLists[ModuleIdentifier], ModuleToSummariesForIndex);
|
|
120
|
710 }
|
|
|
711
|
|
|
712 /**
|
|
|
713 * Emit the list of files needed for importing into module.
|
|
|
714 */
|
|
147
|
715 void ThinLTOCodeGenerator::emitImports(Module &TheModule, StringRef OutputName,
|
|
|
716 ModuleSummaryIndex &Index,
|
|
|
717 const lto::InputFile &File) {
|
|
120
|
718 auto ModuleCount = Index.modulePaths().size();
|
|
147
|
719 auto ModuleIdentifier = TheModule.getModuleIdentifier();
|
|
120
|
720
|
|
|
721 // Collect for each module the list of function it defines (GUID -> Summary).
|
|
|
722 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
|
|
|
723 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
|
|
|
724
|
|
147
|
725 // Convert the preserved symbols set from string to GUID
|
|
|
726 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
|
|
|
727 PreservedSymbols, Triple(TheModule.getTargetTriple()));
|
|
|
728
|
|
|
729 addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
|
|
|
730
|
|
|
731 // Compute "dead" symbols, we don't want to import/export these!
|
|
|
732 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
|
|
|
733
|
|
120
|
734 // Generate import/export list
|
|
|
735 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
|
|
|
736 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
|
|
|
737 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
|
|
|
738 ExportLists);
|
|
|
739
|
|
147
|
740 std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
|
|
|
741 llvm::gatherImportedSummariesForModule(
|
|
|
742 ModuleIdentifier, ModuleToDefinedGVSummaries,
|
|
|
743 ImportLists[ModuleIdentifier], ModuleToSummariesForIndex);
|
|
|
744
|
|
120
|
745 std::error_code EC;
|
|
147
|
746 if ((EC = EmitImportsFiles(ModuleIdentifier, OutputName,
|
|
|
747 ModuleToSummariesForIndex)))
|
|
120
|
748 report_fatal_error(Twine("Failed to open ") + OutputName +
|
|
|
749 " to save imports lists\n");
|
|
|
750 }
|
|
|
751
|
|
|
752 /**
|
|
147
|
753 * Perform internalization. Runs promote and internalization together.
|
|
|
754 * Index is updated to reflect linkage changes.
|
|
120
|
755 */
|
|
|
756 void ThinLTOCodeGenerator::internalize(Module &TheModule,
|
|
147
|
757 ModuleSummaryIndex &Index,
|
|
|
758 const lto::InputFile &File) {
|
|
120
|
759 initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
|
|
|
760 auto ModuleCount = Index.modulePaths().size();
|
|
|
761 auto ModuleIdentifier = TheModule.getModuleIdentifier();
|
|
|
762
|
|
|
763 // Convert the preserved symbols set from string to GUID
|
|
|
764 auto GUIDPreservedSymbols =
|
|
|
765 computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple);
|
|
|
766
|
|
147
|
767 addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
|
|
|
768
|
|
120
|
769 // Collect for each module the list of function it defines (GUID -> Summary).
|
|
|
770 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
|
|
|
771 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
|
|
|
772
|
|
121
|
773 // Compute "dead" symbols, we don't want to import/export these!
|
|
134
|
774 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
|
|
121
|
775
|
|
120
|
776 // Generate import/export list
|
|
|
777 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
|
|
|
778 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
|
|
|
779 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
|
|
|
780 ExportLists);
|
|
|
781 auto &ExportList = ExportLists[ModuleIdentifier];
|
|
|
782
|
|
|
783 // Be friendly and don't nuke totally the module when the client didn't
|
|
|
784 // supply anything to preserve.
|
|
|
785 if (ExportList.empty() && GUIDPreservedSymbols.empty())
|
|
|
786 return;
|
|
|
787
|
|
147
|
788 // Resolve prevailing symbols
|
|
|
789 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
|
|
|
790 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols);
|
|
|
791
|
|
|
792 // Promote the exported values in the index, so that they are promoted
|
|
|
793 // in the module.
|
|
|
794 internalizeAndPromoteInIndex(ExportLists, GUIDPreservedSymbols, Index);
|
|
|
795
|
|
|
796 promoteModule(TheModule, Index);
|
|
|
797
|
|
120
|
798 // Internalization
|
|
147
|
799 thinLTOResolvePrevailingInModule(
|
|
|
800 TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]);
|
|
|
801
|
|
120
|
802 thinLTOInternalizeModule(TheModule,
|
|
|
803 ModuleToDefinedGVSummaries[ModuleIdentifier]);
|
|
|
804 }
|
|
|
805
|
|
|
806 /**
|
|
|
807 * Perform post-importing ThinLTO optimizations.
|
|
|
808 */
|
|
|
809 void ThinLTOCodeGenerator::optimize(Module &TheModule) {
|
|
|
810 initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
|
|
|
811
|
|
|
812 // Optimize now
|
|
121
|
813 optimizeModule(TheModule, *TMBuilder.create(), OptLevel, Freestanding);
|
|
120
|
814 }
|
|
|
815
|
|
121
|
816 /// Write out the generated object file, either from CacheEntryPath or from
|
|
|
817 /// OutputBuffer, preferring hard-link when possible.
|
|
|
818 /// Returns the path to the generated file in SavedObjectsDirectoryPath.
|
|
147
|
819 std::string
|
|
|
820 ThinLTOCodeGenerator::writeGeneratedObject(int count, StringRef CacheEntryPath,
|
|
|
821 const MemoryBuffer &OutputBuffer) {
|
|
|
822 auto ArchName = TMBuilder.TheTriple.getArchName();
|
|
121
|
823 SmallString<128> OutputPath(SavedObjectsDirectoryPath);
|
|
147
|
824 llvm::sys::path::append(OutputPath,
|
|
|
825 Twine(count) + "." + ArchName + ".thinlto.o");
|
|
121
|
826 OutputPath.c_str(); // Ensure the string is null terminated.
|
|
|
827 if (sys::fs::exists(OutputPath))
|
|
|
828 sys::fs::remove(OutputPath);
|
|
|
829
|
|
|
830 // We don't return a memory buffer to the linker, just a list of files.
|
|
|
831 if (!CacheEntryPath.empty()) {
|
|
|
832 // Cache is enabled, hard-link the entry (or copy if hard-link fails).
|
|
|
833 auto Err = sys::fs::create_hard_link(CacheEntryPath, OutputPath);
|
|
|
834 if (!Err)
|
|
|
835 return OutputPath.str();
|
|
|
836 // Hard linking failed, try to copy.
|
|
|
837 Err = sys::fs::copy_file(CacheEntryPath, OutputPath);
|
|
|
838 if (!Err)
|
|
|
839 return OutputPath.str();
|
|
|
840 // Copy failed (could be because the CacheEntry was removed from the cache
|
|
|
841 // in the meantime by another process), fall back and try to write down the
|
|
|
842 // buffer to the output.
|
|
|
843 errs() << "error: can't link or copy from cached entry '" << CacheEntryPath
|
|
|
844 << "' to '" << OutputPath << "'\n";
|
|
|
845 }
|
|
|
846 // No cache entry, just write out the buffer.
|
|
|
847 std::error_code Err;
|
|
147
|
848 raw_fd_ostream OS(OutputPath, Err, sys::fs::OF_None);
|
|
121
|
849 if (Err)
|
|
|
850 report_fatal_error("Can't open output '" + OutputPath + "'\n");
|
|
|
851 OS << OutputBuffer.getBuffer();
|
|
|
852 return OutputPath.str();
|
|
|
853 }
|
|
|
854
|
|
120
|
855 // Main entry point for the ThinLTO processing
|
|
|
856 void ThinLTOCodeGenerator::run() {
|
|
121
|
857 // Prepare the resulting object vector
|
|
|
858 assert(ProducedBinaries.empty() && "The generator should not be reused");
|
|
|
859 if (SavedObjectsDirectoryPath.empty())
|
|
|
860 ProducedBinaries.resize(Modules.size());
|
|
|
861 else {
|
|
|
862 sys::fs::create_directories(SavedObjectsDirectoryPath);
|
|
|
863 bool IsDir;
|
|
|
864 sys::fs::is_directory(SavedObjectsDirectoryPath, IsDir);
|
|
|
865 if (!IsDir)
|
|
|
866 report_fatal_error("Unexistent dir: '" + SavedObjectsDirectoryPath + "'");
|
|
|
867 ProducedBinaryFiles.resize(Modules.size());
|
|
|
868 }
|
|
|
869
|
|
120
|
870 if (CodeGenOnly) {
|
|
|
871 // Perform only parallel codegen and return.
|
|
|
872 ThreadPool Pool;
|
|
|
873 int count = 0;
|
|
147
|
874 for (auto &Mod : Modules) {
|
|
120
|
875 Pool.async([&](int count) {
|
|
|
876 LLVMContext Context;
|
|
|
877 Context.setDiscardValueNames(LTODiscardValueNames);
|
|
|
878
|
|
|
879 // Parse module now
|
|
147
|
880 auto TheModule = loadModuleFromInput(Mod.get(), Context, false,
|
|
|
881 /*IsImporting*/ false);
|
|
120
|
882
|
|
|
883 // CodeGen
|
|
147
|
884 auto OutputBuffer = codegenModule(*TheModule, *TMBuilder.create());
|
|
121
|
885 if (SavedObjectsDirectoryPath.empty())
|
|
|
886 ProducedBinaries[count] = std::move(OutputBuffer);
|
|
|
887 else
|
|
147
|
888 ProducedBinaryFiles[count] =
|
|
|
889 writeGeneratedObject(count, "", *OutputBuffer);
|
|
120
|
890 }, count++);
|
|
|
891 }
|
|
|
892
|
|
|
893 return;
|
|
|
894 }
|
|
|
895
|
|
|
896 // Sequential linking phase
|
|
|
897 auto Index = linkCombinedIndex();
|
|
|
898
|
|
|
899 // Save temps: index.
|
|
|
900 if (!SaveTempsDir.empty()) {
|
|
|
901 auto SaveTempPath = SaveTempsDir + "index.bc";
|
|
|
902 std::error_code EC;
|
|
147
|
903 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
|
|
120
|
904 if (EC)
|
|
|
905 report_fatal_error(Twine("Failed to open ") + SaveTempPath +
|
|
|
906 " to save optimized bitcode\n");
|
|
|
907 WriteIndexToFile(*Index, OS);
|
|
|
908 }
|
|
|
909
|
|
|
910
|
|
|
911 // Prepare the module map.
|
|
|
912 auto ModuleMap = generateModuleMap(Modules);
|
|
|
913 auto ModuleCount = Modules.size();
|
|
|
914
|
|
|
915 // Collect for each module the list of function it defines (GUID -> Summary).
|
|
|
916 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
|
|
|
917 Index->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
|
|
|
918
|
|
121
|
919 // Convert the preserved symbols set from string to GUID, this is needed for
|
|
|
920 // computing the caching hash and the internalization.
|
|
|
921 auto GUIDPreservedSymbols =
|
|
|
922 computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple);
|
|
|
923
|
|
147
|
924 // Add used symbol from inputs to the preserved symbols.
|
|
|
925 for (const auto &M : Modules)
|
|
|
926 addUsedSymbolToPreservedGUID(*M, GUIDPreservedSymbols);
|
|
|
927
|
|
121
|
928 // Compute "dead" symbols, we don't want to import/export these!
|
|
134
|
929 computeDeadSymbolsInIndex(*Index, GUIDPreservedSymbols);
|
|
121
|
930
|
|
147
|
931 // Synthesize entry counts for functions in the combined index.
|
|
|
932 computeSyntheticCounts(*Index);
|
|
|
933
|
|
120
|
934 // Collect the import/export lists for all modules from the call-graph in the
|
|
|
935 // combined index.
|
|
|
936 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
|
|
|
937 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
|
|
|
938 ComputeCrossModuleImport(*Index, ModuleToDefinedGVSummaries, ImportLists,
|
|
|
939 ExportLists);
|
|
|
940
|
|
|
941 // We use a std::map here to be able to have a defined ordering when
|
|
|
942 // producing a hash for the cache entry.
|
|
|
943 // FIXME: we should be able to compute the caching hash for the entry based
|
|
|
944 // on the index, and nuke this map.
|
|
|
945 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
|
|
|
946
|
|
147
|
947 // Resolve prevailing symbols, this has to be computed early because it
|
|
120
|
948 // impacts the caching.
|
|
147
|
949 resolvePrevailingInIndex(*Index, ResolvedODR, GUIDPreservedSymbols);
|
|
120
|
950
|
|
|
951 // Use global summary-based analysis to identify symbols that can be
|
|
|
952 // internalized (because they aren't exported or preserved as per callback).
|
|
|
953 // Changes are made in the index, consumed in the ThinLTO backends.
|
|
134
|
954 internalizeAndPromoteInIndex(ExportLists, GUIDPreservedSymbols, *Index);
|
|
120
|
955
|
|
147
|
956 // Make sure that every module has an entry in the ExportLists, ImportList,
|
|
|
957 // GVSummary and ResolvedODR maps to enable threaded access to these maps
|
|
|
958 // below.
|
|
|
959 for (auto &Module : Modules) {
|
|
|
960 auto ModuleIdentifier = Module->getName();
|
|
|
961 ExportLists[ModuleIdentifier];
|
|
|
962 ImportLists[ModuleIdentifier];
|
|
|
963 ResolvedODR[ModuleIdentifier];
|
|
|
964 ModuleToDefinedGVSummaries[ModuleIdentifier];
|
|
120
|
965 }
|
|
|
966
|
|
|
967 // Compute the ordering we will process the inputs: the rough heuristic here
|
|
|
968 // is to sort them per size so that the largest module get schedule as soon as
|
|
|
969 // possible. This is purely a compile-time optimization.
|
|
|
970 std::vector<int> ModulesOrdering;
|
|
|
971 ModulesOrdering.resize(Modules.size());
|
|
|
972 std::iota(ModulesOrdering.begin(), ModulesOrdering.end(), 0);
|
|
147
|
973 llvm::sort(ModulesOrdering, [&](int LeftIndex, int RightIndex) {
|
|
|
974 auto LSize =
|
|
|
975 Modules[LeftIndex]->getSingleBitcodeModule().getBuffer().size();
|
|
|
976 auto RSize =
|
|
|
977 Modules[RightIndex]->getSingleBitcodeModule().getBuffer().size();
|
|
|
978 return LSize > RSize;
|
|
|
979 });
|
|
120
|
980
|
|
|
981 // Parallel optimizer + codegen
|
|
|
982 {
|
|
|
983 ThreadPool Pool(ThreadCount);
|
|
|
984 for (auto IndexCount : ModulesOrdering) {
|
|
147
|
985 auto &Mod = Modules[IndexCount];
|
|
120
|
986 Pool.async([&](int count) {
|
|
147
|
987 auto ModuleIdentifier = Mod->getName();
|
|
120
|
988 auto &ExportList = ExportLists[ModuleIdentifier];
|
|
|
989
|
|
147
|
990 auto &DefinedGVSummaries = ModuleToDefinedGVSummaries[ModuleIdentifier];
|
|
120
|
991
|
|
|
992 // The module may be cached, this helps handling it.
|
|
|
993 ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier,
|
|
|
994 ImportLists[ModuleIdentifier], ExportList,
|
|
|
995 ResolvedODR[ModuleIdentifier],
|
|
147
|
996 DefinedGVSummaries, OptLevel, Freestanding,
|
|
|
997 TMBuilder);
|
|
121
|
998 auto CacheEntryPath = CacheEntry.getEntryPath();
|
|
120
|
999
|
|
|
1000 {
|
|
|
1001 auto ErrOrBuffer = CacheEntry.tryLoadingBuffer();
|
|
147
|
1002 LLVM_DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss")
|
|
|
1003 << " '" << CacheEntryPath << "' for buffer "
|
|
|
1004 << count << " " << ModuleIdentifier << "\n");
|
|
120
|
1005
|
|
|
1006 if (ErrOrBuffer) {
|
|
|
1007 // Cache Hit!
|
|
121
|
1008 if (SavedObjectsDirectoryPath.empty())
|
|
|
1009 ProducedBinaries[count] = std::move(ErrOrBuffer.get());
|
|
|
1010 else
|
|
|
1011 ProducedBinaryFiles[count] = writeGeneratedObject(
|
|
147
|
1012 count, CacheEntryPath, *ErrOrBuffer.get());
|
|
120
|
1013 return;
|
|
|
1014 }
|
|
|
1015 }
|
|
|
1016
|
|
|
1017 LLVMContext Context;
|
|
|
1018 Context.setDiscardValueNames(LTODiscardValueNames);
|
|
|
1019 Context.enableDebugTypeODRUniquing();
|
|
121
|
1020 auto DiagFileOrErr = lto::setupOptimizationRemarks(
|
|
147
|
1021 Context, RemarksFilename, RemarksPasses, RemarksFormat,
|
|
|
1022 RemarksWithHotness, count);
|
|
120
|
1023 if (!DiagFileOrErr) {
|
|
|
1024 errs() << "Error: " << toString(DiagFileOrErr.takeError()) << "\n";
|
|
|
1025 report_fatal_error("ThinLTO: Can't get an output file for the "
|
|
|
1026 "remarks");
|
|
|
1027 }
|
|
|
1028
|
|
|
1029 // Parse module now
|
|
147
|
1030 auto TheModule = loadModuleFromInput(Mod.get(), Context, false,
|
|
|
1031 /*IsImporting*/ false);
|
|
120
|
1032
|
|
|
1033 // Save temps: original file.
|
|
|
1034 saveTempBitcode(*TheModule, SaveTempsDir, count, ".0.original.bc");
|
|
|
1035
|
|
|
1036 auto &ImportList = ImportLists[ModuleIdentifier];
|
|
|
1037 // Run the main process now, and generates a binary
|
|
|
1038 auto OutputBuffer = ProcessThinLTOModule(
|
|
|
1039 *TheModule, *Index, ModuleMap, *TMBuilder.create(), ImportList,
|
|
|
1040 ExportList, GUIDPreservedSymbols,
|
|
|
1041 ModuleToDefinedGVSummaries[ModuleIdentifier], CacheOptions,
|
|
121
|
1042 DisableCodeGen, SaveTempsDir, Freestanding, OptLevel, count);
|
|
|
1043
|
|
|
1044 // Commit to the cache (if enabled)
|
|
|
1045 CacheEntry.write(*OutputBuffer);
|
|
120
|
1046
|
|
121
|
1047 if (SavedObjectsDirectoryPath.empty()) {
|
|
|
1048 // We need to generated a memory buffer for the linker.
|
|
|
1049 if (!CacheEntryPath.empty()) {
|
|
147
|
1050 // When cache is enabled, reload from the cache if possible.
|
|
|
1051 // Releasing the buffer from the heap and reloading it from the
|
|
|
1052 // cache file with mmap helps us to lower memory pressure.
|
|
|
1053 // The freed memory can be used for the next input file.
|
|
|
1054 // The final binary link will read from the VFS cache (hopefully!)
|
|
|
1055 // or from disk (if the memory pressure was too high).
|
|
121
|
1056 auto ReloadedBufferOrErr = CacheEntry.tryLoadingBuffer();
|
|
|
1057 if (auto EC = ReloadedBufferOrErr.getError()) {
|
|
147
|
1058 // On error, keep the preexisting buffer and print a diagnostic.
|
|
121
|
1059 errs() << "error: can't reload cached file '" << CacheEntryPath
|
|
|
1060 << "': " << EC.message() << "\n";
|
|
|
1061 } else {
|
|
|
1062 OutputBuffer = std::move(*ReloadedBufferOrErr);
|
|
|
1063 }
|
|
|
1064 }
|
|
|
1065 ProducedBinaries[count] = std::move(OutputBuffer);
|
|
|
1066 return;
|
|
|
1067 }
|
|
|
1068 ProducedBinaryFiles[count] = writeGeneratedObject(
|
|
147
|
1069 count, CacheEntryPath, *OutputBuffer);
|
|
120
|
1070 }, IndexCount);
|
|
|
1071 }
|
|
|
1072 }
|
|
|
1073
|
|
121
|
1074 pruneCache(CacheOptions.Path, CacheOptions.Policy);
|
|
120
|
1075
|
|
|
1076 // If statistics were requested, print them out now.
|
|
|
1077 if (llvm::AreStatisticsEnabled())
|
|
|
1078 llvm::PrintStatistics();
|
|
121
|
1079 reportAndResetTimings();
|
|
120
|
1080 }
|
