|
120
|
1 //===- ModuleSummaryAnalysis.cpp - Module summary index builder -----------===//
|
|
|
2 //
|
|
|
3 // The LLVM Compiler Infrastructure
|
|
|
4 //
|
|
|
5 // This file is distributed under the University of Illinois Open Source
|
|
|
6 // License. See LICENSE.TXT for details.
|
|
|
7 //
|
|
|
8 //===----------------------------------------------------------------------===//
|
|
|
9 //
|
|
|
10 // This pass builds a ModuleSummaryIndex object for the module, to be written
|
|
|
11 // to bitcode or LLVM assembly.
|
|
|
12 //
|
|
|
13 //===----------------------------------------------------------------------===//
|
|
|
14
|
|
|
15 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
|
|
121
|
16 #include "llvm/ADT/ArrayRef.h"
|
|
|
17 #include "llvm/ADT/DenseSet.h"
|
|
|
18 #include "llvm/ADT/MapVector.h"
|
|
|
19 #include "llvm/ADT/STLExtras.h"
|
|
|
20 #include "llvm/ADT/SetVector.h"
|
|
|
21 #include "llvm/ADT/SmallPtrSet.h"
|
|
|
22 #include "llvm/ADT/SmallVector.h"
|
|
|
23 #include "llvm/ADT/StringRef.h"
|
|
120
|
24 #include "llvm/Analysis/BlockFrequencyInfo.h"
|
|
|
25 #include "llvm/Analysis/BranchProbabilityInfo.h"
|
|
|
26 #include "llvm/Analysis/IndirectCallPromotionAnalysis.h"
|
|
|
27 #include "llvm/Analysis/LoopInfo.h"
|
|
|
28 #include "llvm/Analysis/ProfileSummaryInfo.h"
|
|
121
|
29 #include "llvm/Analysis/TypeMetadataUtils.h"
|
|
|
30 #include "llvm/IR/Attributes.h"
|
|
|
31 #include "llvm/IR/BasicBlock.h"
|
|
120
|
32 #include "llvm/IR/CallSite.h"
|
|
121
|
33 #include "llvm/IR/Constant.h"
|
|
|
34 #include "llvm/IR/Constants.h"
|
|
120
|
35 #include "llvm/IR/Dominators.h"
|
|
121
|
36 #include "llvm/IR/Function.h"
|
|
|
37 #include "llvm/IR/GlobalAlias.h"
|
|
|
38 #include "llvm/IR/GlobalValue.h"
|
|
|
39 #include "llvm/IR/GlobalVariable.h"
|
|
|
40 #include "llvm/IR/Instructions.h"
|
|
120
|
41 #include "llvm/IR/IntrinsicInst.h"
|
|
121
|
42 #include "llvm/IR/Intrinsics.h"
|
|
|
43 #include "llvm/IR/Metadata.h"
|
|
|
44 #include "llvm/IR/Module.h"
|
|
|
45 #include "llvm/IR/ModuleSummaryIndex.h"
|
|
|
46 #include "llvm/IR/Use.h"
|
|
|
47 #include "llvm/IR/User.h"
|
|
|
48 #include "llvm/Object/ModuleSymbolTable.h"
|
|
|
49 #include "llvm/Object/SymbolicFile.h"
|
|
120
|
50 #include "llvm/Pass.h"
|
|
121
|
51 #include "llvm/Support/Casting.h"
|
|
|
52 #include <algorithm>
|
|
|
53 #include <cassert>
|
|
|
54 #include <cstdint>
|
|
|
55 #include <vector>
|
|
|
56
|
|
120
|
57 using namespace llvm;
|
|
|
58
|
|
|
59 #define DEBUG_TYPE "module-summary-analysis"
|
|
|
60
|
|
|
61 // Walk through the operands of a given User via worklist iteration and populate
|
|
|
62 // the set of GlobalValue references encountered. Invoked either on an
|
|
|
63 // Instruction or a GlobalVariable (which walks its initializer).
|
|
121
|
64 static void findRefEdges(ModuleSummaryIndex &Index, const User *CurUser,
|
|
|
65 SetVector<ValueInfo> &RefEdges,
|
|
120
|
66 SmallPtrSet<const User *, 8> &Visited) {
|
|
|
67 SmallVector<const User *, 32> Worklist;
|
|
|
68 Worklist.push_back(CurUser);
|
|
|
69
|
|
|
70 while (!Worklist.empty()) {
|
|
|
71 const User *U = Worklist.pop_back_val();
|
|
|
72
|
|
|
73 if (!Visited.insert(U).second)
|
|
|
74 continue;
|
|
|
75
|
|
|
76 ImmutableCallSite CS(U);
|
|
|
77
|
|
|
78 for (const auto &OI : U->operands()) {
|
|
|
79 const User *Operand = dyn_cast<User>(OI);
|
|
|
80 if (!Operand)
|
|
|
81 continue;
|
|
|
82 if (isa<BlockAddress>(Operand))
|
|
|
83 continue;
|
|
121
|
84 if (auto *GV = dyn_cast<GlobalValue>(Operand)) {
|
|
120
|
85 // We have a reference to a global value. This should be added to
|
|
|
86 // the reference set unless it is a callee. Callees are handled
|
|
|
87 // specially by WriteFunction and are added to a separate list.
|
|
|
88 if (!(CS && CS.isCallee(&OI)))
|
|
121
|
89 RefEdges.insert(Index.getOrInsertValueInfo(GV));
|
|
120
|
90 continue;
|
|
|
91 }
|
|
|
92 Worklist.push_back(Operand);
|
|
|
93 }
|
|
|
94 }
|
|
|
95 }
|
|
|
96
|
|
|
97 static CalleeInfo::HotnessType getHotness(uint64_t ProfileCount,
|
|
|
98 ProfileSummaryInfo *PSI) {
|
|
|
99 if (!PSI)
|
|
|
100 return CalleeInfo::HotnessType::Unknown;
|
|
|
101 if (PSI->isHotCount(ProfileCount))
|
|
|
102 return CalleeInfo::HotnessType::Hot;
|
|
|
103 if (PSI->isColdCount(ProfileCount))
|
|
|
104 return CalleeInfo::HotnessType::Cold;
|
|
|
105 return CalleeInfo::HotnessType::None;
|
|
|
106 }
|
|
|
107
|
|
121
|
108 static bool isNonRenamableLocal(const GlobalValue &GV) {
|
|
|
109 return GV.hasSection() && GV.hasLocalLinkage();
|
|
|
110 }
|
|
|
111
|
|
|
112 /// Determine whether this call has all constant integer arguments (excluding
|
|
|
113 /// "this") and summarize it to VCalls or ConstVCalls as appropriate.
|
|
|
114 static void addVCallToSet(DevirtCallSite Call, GlobalValue::GUID Guid,
|
|
|
115 SetVector<FunctionSummary::VFuncId> &VCalls,
|
|
|
116 SetVector<FunctionSummary::ConstVCall> &ConstVCalls) {
|
|
|
117 std::vector<uint64_t> Args;
|
|
|
118 // Start from the second argument to skip the "this" pointer.
|
|
|
119 for (auto &Arg : make_range(Call.CS.arg_begin() + 1, Call.CS.arg_end())) {
|
|
|
120 auto *CI = dyn_cast<ConstantInt>(Arg);
|
|
|
121 if (!CI || CI->getBitWidth() > 64) {
|
|
|
122 VCalls.insert({Guid, Call.Offset});
|
|
|
123 return;
|
|
|
124 }
|
|
|
125 Args.push_back(CI->getZExtValue());
|
|
|
126 }
|
|
|
127 ConstVCalls.insert({{Guid, Call.Offset}, std::move(Args)});
|
|
|
128 }
|
|
|
129
|
|
|
130 /// If this intrinsic call requires that we add information to the function
|
|
|
131 /// summary, do so via the non-constant reference arguments.
|
|
|
132 static void addIntrinsicToSummary(
|
|
|
133 const CallInst *CI, SetVector<GlobalValue::GUID> &TypeTests,
|
|
|
134 SetVector<FunctionSummary::VFuncId> &TypeTestAssumeVCalls,
|
|
|
135 SetVector<FunctionSummary::VFuncId> &TypeCheckedLoadVCalls,
|
|
|
136 SetVector<FunctionSummary::ConstVCall> &TypeTestAssumeConstVCalls,
|
|
|
137 SetVector<FunctionSummary::ConstVCall> &TypeCheckedLoadConstVCalls) {
|
|
|
138 switch (CI->getCalledFunction()->getIntrinsicID()) {
|
|
|
139 case Intrinsic::type_test: {
|
|
|
140 auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(1));
|
|
|
141 auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());
|
|
|
142 if (!TypeId)
|
|
|
143 break;
|
|
|
144 GlobalValue::GUID Guid = GlobalValue::getGUID(TypeId->getString());
|
|
|
145
|
|
|
146 // Produce a summary from type.test intrinsics. We only summarize type.test
|
|
|
147 // intrinsics that are used other than by an llvm.assume intrinsic.
|
|
|
148 // Intrinsics that are assumed are relevant only to the devirtualization
|
|
|
149 // pass, not the type test lowering pass.
|
|
|
150 bool HasNonAssumeUses = llvm::any_of(CI->uses(), [](const Use &CIU) {
|
|
|
151 auto *AssumeCI = dyn_cast<CallInst>(CIU.getUser());
|
|
|
152 if (!AssumeCI)
|
|
|
153 return true;
|
|
|
154 Function *F = AssumeCI->getCalledFunction();
|
|
|
155 return !F || F->getIntrinsicID() != Intrinsic::assume;
|
|
|
156 });
|
|
|
157 if (HasNonAssumeUses)
|
|
|
158 TypeTests.insert(Guid);
|
|
|
159
|
|
|
160 SmallVector<DevirtCallSite, 4> DevirtCalls;
|
|
|
161 SmallVector<CallInst *, 4> Assumes;
|
|
|
162 findDevirtualizableCallsForTypeTest(DevirtCalls, Assumes, CI);
|
|
|
163 for (auto &Call : DevirtCalls)
|
|
|
164 addVCallToSet(Call, Guid, TypeTestAssumeVCalls,
|
|
|
165 TypeTestAssumeConstVCalls);
|
|
|
166
|
|
|
167 break;
|
|
|
168 }
|
|
|
169
|
|
|
170 case Intrinsic::type_checked_load: {
|
|
|
171 auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(2));
|
|
|
172 auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());
|
|
|
173 if (!TypeId)
|
|
|
174 break;
|
|
|
175 GlobalValue::GUID Guid = GlobalValue::getGUID(TypeId->getString());
|
|
|
176
|
|
|
177 SmallVector<DevirtCallSite, 4> DevirtCalls;
|
|
|
178 SmallVector<Instruction *, 4> LoadedPtrs;
|
|
|
179 SmallVector<Instruction *, 4> Preds;
|
|
|
180 bool HasNonCallUses = false;
|
|
|
181 findDevirtualizableCallsForTypeCheckedLoad(DevirtCalls, LoadedPtrs, Preds,
|
|
|
182 HasNonCallUses, CI);
|
|
|
183 // Any non-call uses of the result of llvm.type.checked.load will
|
|
|
184 // prevent us from optimizing away the llvm.type.test.
|
|
|
185 if (HasNonCallUses)
|
|
|
186 TypeTests.insert(Guid);
|
|
|
187 for (auto &Call : DevirtCalls)
|
|
|
188 addVCallToSet(Call, Guid, TypeCheckedLoadVCalls,
|
|
|
189 TypeCheckedLoadConstVCalls);
|
|
|
190
|
|
|
191 break;
|
|
|
192 }
|
|
|
193 default:
|
|
|
194 break;
|
|
|
195 }
|
|
|
196 }
|
|
|
197
|
|
|
198 static void
|
|
|
199 computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
|
|
|
200 const Function &F, BlockFrequencyInfo *BFI,
|
|
|
201 ProfileSummaryInfo *PSI, bool HasLocalsInUsedOrAsm,
|
|
|
202 DenseSet<GlobalValue::GUID> &CantBePromoted) {
|
|
120
|
203 // Summary not currently supported for anonymous functions, they should
|
|
|
204 // have been named.
|
|
|
205 assert(F.hasName());
|
|
|
206
|
|
|
207 unsigned NumInsts = 0;
|
|
|
208 // Map from callee ValueId to profile count. Used to accumulate profile
|
|
|
209 // counts for all static calls to a given callee.
|
|
121
|
210 MapVector<ValueInfo, CalleeInfo> CallGraphEdges;
|
|
|
211 SetVector<ValueInfo> RefEdges;
|
|
|
212 SetVector<GlobalValue::GUID> TypeTests;
|
|
|
213 SetVector<FunctionSummary::VFuncId> TypeTestAssumeVCalls,
|
|
|
214 TypeCheckedLoadVCalls;
|
|
|
215 SetVector<FunctionSummary::ConstVCall> TypeTestAssumeConstVCalls,
|
|
|
216 TypeCheckedLoadConstVCalls;
|
|
120
|
217 ICallPromotionAnalysis ICallAnalysis;
|
|
121
|
218 SmallPtrSet<const User *, 8> Visited;
|
|
|
219
|
|
|
220 // Add personality function, prefix data and prologue data to function's ref
|
|
|
221 // list.
|
|
|
222 findRefEdges(Index, &F, RefEdges, Visited);
|
|
120
|
223
|
|
|
224 bool HasInlineAsmMaybeReferencingInternal = false;
|
|
|
225 for (const BasicBlock &BB : F)
|
|
|
226 for (const Instruction &I : BB) {
|
|
|
227 if (isa<DbgInfoIntrinsic>(I))
|
|
|
228 continue;
|
|
|
229 ++NumInsts;
|
|
121
|
230 findRefEdges(Index, &I, RefEdges, Visited);
|
|
120
|
231 auto CS = ImmutableCallSite(&I);
|
|
|
232 if (!CS)
|
|
|
233 continue;
|
|
|
234
|
|
|
235 const auto *CI = dyn_cast<CallInst>(&I);
|
|
|
236 // Since we don't know exactly which local values are referenced in inline
|
|
|
237 // assembly, conservatively mark the function as possibly referencing
|
|
|
238 // a local value from inline assembly to ensure we don't export a
|
|
|
239 // reference (which would require renaming and promotion of the
|
|
|
240 // referenced value).
|
|
121
|
241 if (HasLocalsInUsedOrAsm && CI && CI->isInlineAsm())
|
|
120
|
242 HasInlineAsmMaybeReferencingInternal = true;
|
|
|
243
|
|
|
244 auto *CalledValue = CS.getCalledValue();
|
|
|
245 auto *CalledFunction = CS.getCalledFunction();
|
|
134
|
246 if (CalledValue && !CalledFunction) {
|
|
|
247 CalledValue = CalledValue->stripPointerCastsNoFollowAliases();
|
|
|
248 // Stripping pointer casts can reveal a called function.
|
|
|
249 CalledFunction = dyn_cast<Function>(CalledValue);
|
|
|
250 }
|
|
120
|
251 // Check if this is an alias to a function. If so, get the
|
|
|
252 // called aliasee for the checks below.
|
|
|
253 if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
|
|
|
254 assert(!CalledFunction && "Expected null called function in callsite for alias");
|
|
|
255 CalledFunction = dyn_cast<Function>(GA->getBaseObject());
|
|
|
256 }
|
|
121
|
257 // Check if this is a direct call to a known function or a known
|
|
|
258 // intrinsic, or an indirect call with profile data.
|
|
120
|
259 if (CalledFunction) {
|
|
121
|
260 if (CI && CalledFunction->isIntrinsic()) {
|
|
|
261 addIntrinsicToSummary(
|
|
|
262 CI, TypeTests, TypeTestAssumeVCalls, TypeCheckedLoadVCalls,
|
|
|
263 TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls);
|
|
120
|
264 continue;
|
|
121
|
265 }
|
|
120
|
266 // We should have named any anonymous globals
|
|
|
267 assert(CalledFunction->hasName());
|
|
121
|
268 auto ScaledCount = PSI->getProfileCount(&I, BFI);
|
|
|
269 auto Hotness = ScaledCount ? getHotness(ScaledCount.getValue(), PSI)
|
|
|
270 : CalleeInfo::HotnessType::Unknown;
|
|
|
271
|
|
120
|
272 // Use the original CalledValue, in case it was an alias. We want
|
|
|
273 // to record the call edge to the alias in that case. Eventually
|
|
|
274 // an alias summary will be created to associate the alias and
|
|
|
275 // aliasee.
|
|
134
|
276 auto &ValueInfo = CallGraphEdges[Index.getOrInsertValueInfo(
|
|
|
277 cast<GlobalValue>(CalledValue))];
|
|
|
278 ValueInfo.updateHotness(Hotness);
|
|
|
279 // Add the relative block frequency to CalleeInfo if there is no profile
|
|
|
280 // information.
|
|
|
281 if (BFI != nullptr && Hotness == CalleeInfo::HotnessType::Unknown) {
|
|
|
282 auto BBFreq = BFI->getBlockFreq(&BB).getFrequency();
|
|
|
283 // FIXME: This might need some scaling to prevent BBFreq values from
|
|
|
284 // being rounded down to 0.
|
|
|
285 auto EntryFreq = BFI->getEntryFreq();
|
|
|
286 // Block frequencies can be directly set for a block and so we need to
|
|
|
287 // handle the case of entry frequency being 0.
|
|
|
288 if (EntryFreq)
|
|
|
289 BBFreq /= EntryFreq;
|
|
|
290 else
|
|
|
291 BBFreq = 0;
|
|
|
292 ValueInfo.updateRelBlockFreq(BBFreq);
|
|
|
293 }
|
|
120
|
294 } else {
|
|
|
295 // Skip inline assembly calls.
|
|
|
296 if (CI && CI->isInlineAsm())
|
|
|
297 continue;
|
|
|
298 // Skip direct calls.
|
|
134
|
299 if (!CalledValue || isa<Constant>(CalledValue))
|
|
120
|
300 continue;
|
|
|
301
|
|
|
302 uint32_t NumVals, NumCandidates;
|
|
|
303 uint64_t TotalCount;
|
|
|
304 auto CandidateProfileData =
|
|
|
305 ICallAnalysis.getPromotionCandidatesForInstruction(
|
|
|
306 &I, NumVals, TotalCount, NumCandidates);
|
|
|
307 for (auto &Candidate : CandidateProfileData)
|
|
121
|
308 CallGraphEdges[Index.getOrInsertValueInfo(Candidate.Value)]
|
|
|
309 .updateHotness(getHotness(Candidate.Count, PSI));
|
|
120
|
310 }
|
|
|
311 }
|
|
|
312
|
|
121
|
313 // Explicit add hot edges to enforce importing for designated GUIDs for
|
|
|
314 // sample PGO, to enable the same inlines as the profiled optimized binary.
|
|
|
315 for (auto &I : F.getImportGUIDs())
|
|
|
316 CallGraphEdges[Index.getOrInsertValueInfo(I)].updateHotness(
|
|
|
317 CalleeInfo::HotnessType::Critical);
|
|
|
318
|
|
|
319 bool NonRenamableLocal = isNonRenamableLocal(F);
|
|
|
320 bool NotEligibleForImport =
|
|
|
321 NonRenamableLocal || HasInlineAsmMaybeReferencingInternal ||
|
|
|
322 // Inliner doesn't handle variadic functions.
|
|
|
323 // FIXME: refactor this to use the same code that inliner is using.
|
|
134
|
324 F.isVarArg() ||
|
|
|
325 // Don't try to import functions with noinline attribute.
|
|
|
326 F.getAttributes().hasFnAttribute(Attribute::NoInline);
|
|
121
|
327 GlobalValueSummary::GVFlags Flags(F.getLinkage(), NotEligibleForImport,
|
|
134
|
328 /* Live = */ false, F.isDSOLocal());
|
|
121
|
329 FunctionSummary::FFlags FunFlags{
|
|
|
330 F.hasFnAttribute(Attribute::ReadNone),
|
|
|
331 F.hasFnAttribute(Attribute::ReadOnly),
|
|
|
332 F.hasFnAttribute(Attribute::NoRecurse),
|
|
|
333 F.returnDoesNotAlias(),
|
|
|
334 };
|
|
|
335 auto FuncSummary = llvm::make_unique<FunctionSummary>(
|
|
|
336 Flags, NumInsts, FunFlags, RefEdges.takeVector(),
|
|
|
337 CallGraphEdges.takeVector(), TypeTests.takeVector(),
|
|
|
338 TypeTestAssumeVCalls.takeVector(), TypeCheckedLoadVCalls.takeVector(),
|
|
|
339 TypeTestAssumeConstVCalls.takeVector(),
|
|
|
340 TypeCheckedLoadConstVCalls.takeVector());
|
|
|
341 if (NonRenamableLocal)
|
|
|
342 CantBePromoted.insert(F.getGUID());
|
|
120
|
343 Index.addGlobalValueSummary(F.getName(), std::move(FuncSummary));
|
|
|
344 }
|
|
|
345
|
|
121
|
346 static void
|
|
|
347 computeVariableSummary(ModuleSummaryIndex &Index, const GlobalVariable &V,
|
|
|
348 DenseSet<GlobalValue::GUID> &CantBePromoted) {
|
|
|
349 SetVector<ValueInfo> RefEdges;
|
|
120
|
350 SmallPtrSet<const User *, 8> Visited;
|
|
121
|
351 findRefEdges(Index, &V, RefEdges, Visited);
|
|
|
352 bool NonRenamableLocal = isNonRenamableLocal(V);
|
|
|
353 GlobalValueSummary::GVFlags Flags(V.getLinkage(), NonRenamableLocal,
|
|
134
|
354 /* Live = */ false, V.isDSOLocal());
|
|
121
|
355 auto GVarSummary =
|
|
|
356 llvm::make_unique<GlobalVarSummary>(Flags, RefEdges.takeVector());
|
|
|
357 if (NonRenamableLocal)
|
|
|
358 CantBePromoted.insert(V.getGUID());
|
|
120
|
359 Index.addGlobalValueSummary(V.getName(), std::move(GVarSummary));
|
|
|
360 }
|
|
|
361
|
|
121
|
362 static void
|
|
|
363 computeAliasSummary(ModuleSummaryIndex &Index, const GlobalAlias &A,
|
|
|
364 DenseSet<GlobalValue::GUID> &CantBePromoted) {
|
|
|
365 bool NonRenamableLocal = isNonRenamableLocal(A);
|
|
|
366 GlobalValueSummary::GVFlags Flags(A.getLinkage(), NonRenamableLocal,
|
|
134
|
367 /* Live = */ false, A.isDSOLocal());
|
|
121
|
368 auto AS = llvm::make_unique<AliasSummary>(Flags);
|
|
120
|
369 auto *Aliasee = A.getBaseObject();
|
|
|
370 auto *AliaseeSummary = Index.getGlobalValueSummary(*Aliasee);
|
|
|
371 assert(AliaseeSummary && "Alias expects aliasee summary to be parsed");
|
|
|
372 AS->setAliasee(AliaseeSummary);
|
|
121
|
373 if (NonRenamableLocal)
|
|
|
374 CantBePromoted.insert(A.getGUID());
|
|
120
|
375 Index.addGlobalValueSummary(A.getName(), std::move(AS));
|
|
|
376 }
|
|
|
377
|
|
121
|
378 // Set LiveRoot flag on entries matching the given value name.
|
|
|
379 static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name) {
|
|
|
380 if (ValueInfo VI = Index.getValueInfo(GlobalValue::getGUID(Name)))
|
|
|
381 for (auto &Summary : VI.getSummaryList())
|
|
|
382 Summary->setLive(true);
|
|
|
383 }
|
|
|
384
|
|
120
|
385 ModuleSummaryIndex llvm::buildModuleSummaryIndex(
|
|
|
386 const Module &M,
|
|
|
387 std::function<BlockFrequencyInfo *(const Function &F)> GetBFICallback,
|
|
|
388 ProfileSummaryInfo *PSI) {
|
|
121
|
389 assert(PSI);
|
|
134
|
390 ModuleSummaryIndex Index(/*IsPerformingAnalysis=*/true);
|
|
120
|
391
|
|
|
392 // Identify the local values in the llvm.used and llvm.compiler.used sets,
|
|
|
393 // which should not be exported as they would then require renaming and
|
|
|
394 // promotion, but we may have opaque uses e.g. in inline asm. We collect them
|
|
|
395 // here because we use this information to mark functions containing inline
|
|
|
396 // assembly calls as not importable.
|
|
|
397 SmallPtrSet<GlobalValue *, 8> LocalsUsed;
|
|
|
398 SmallPtrSet<GlobalValue *, 8> Used;
|
|
|
399 // First collect those in the llvm.used set.
|
|
|
400 collectUsedGlobalVariables(M, Used, /*CompilerUsed*/ false);
|
|
|
401 // Next collect those in the llvm.compiler.used set.
|
|
|
402 collectUsedGlobalVariables(M, Used, /*CompilerUsed*/ true);
|
|
121
|
403 DenseSet<GlobalValue::GUID> CantBePromoted;
|
|
120
|
404 for (auto *V : Used) {
|
|
121
|
405 if (V->hasLocalLinkage()) {
|
|
120
|
406 LocalsUsed.insert(V);
|
|
121
|
407 CantBePromoted.insert(V->getGUID());
|
|
|
408 }
|
|
|
409 }
|
|
|
410
|
|
|
411 bool HasLocalInlineAsmSymbol = false;
|
|
|
412 if (!M.getModuleInlineAsm().empty()) {
|
|
|
413 // Collect the local values defined by module level asm, and set up
|
|
|
414 // summaries for these symbols so that they can be marked as NoRename,
|
|
|
415 // to prevent export of any use of them in regular IR that would require
|
|
|
416 // renaming within the module level asm. Note we don't need to create a
|
|
|
417 // summary for weak or global defs, as they don't need to be flagged as
|
|
|
418 // NoRename, and defs in module level asm can't be imported anyway.
|
|
|
419 // Also, any values used but not defined within module level asm should
|
|
|
420 // be listed on the llvm.used or llvm.compiler.used global and marked as
|
|
|
421 // referenced from there.
|
|
|
422 ModuleSymbolTable::CollectAsmSymbols(
|
|
|
423 M, [&](StringRef Name, object::BasicSymbolRef::Flags Flags) {
|
|
|
424 // Symbols not marked as Weak or Global are local definitions.
|
|
|
425 if (Flags & (object::BasicSymbolRef::SF_Weak |
|
|
|
426 object::BasicSymbolRef::SF_Global))
|
|
|
427 return;
|
|
|
428 HasLocalInlineAsmSymbol = true;
|
|
|
429 GlobalValue *GV = M.getNamedValue(Name);
|
|
|
430 if (!GV)
|
|
|
431 return;
|
|
|
432 assert(GV->isDeclaration() && "Def in module asm already has definition");
|
|
|
433 GlobalValueSummary::GVFlags GVFlags(GlobalValue::InternalLinkage,
|
|
|
434 /* NotEligibleToImport = */ true,
|
|
134
|
435 /* Live = */ true,
|
|
|
436 /* Local */ GV->isDSOLocal());
|
|
121
|
437 CantBePromoted.insert(GlobalValue::getGUID(Name));
|
|
|
438 // Create the appropriate summary type.
|
|
|
439 if (Function *F = dyn_cast<Function>(GV)) {
|
|
|
440 std::unique_ptr<FunctionSummary> Summary =
|
|
|
441 llvm::make_unique<FunctionSummary>(
|
|
|
442 GVFlags, 0,
|
|
|
443 FunctionSummary::FFlags{
|
|
|
444 F->hasFnAttribute(Attribute::ReadNone),
|
|
|
445 F->hasFnAttribute(Attribute::ReadOnly),
|
|
|
446 F->hasFnAttribute(Attribute::NoRecurse),
|
|
|
447 F->returnDoesNotAlias()},
|
|
|
448 ArrayRef<ValueInfo>{}, ArrayRef<FunctionSummary::EdgeTy>{},
|
|
|
449 ArrayRef<GlobalValue::GUID>{},
|
|
|
450 ArrayRef<FunctionSummary::VFuncId>{},
|
|
|
451 ArrayRef<FunctionSummary::VFuncId>{},
|
|
|
452 ArrayRef<FunctionSummary::ConstVCall>{},
|
|
|
453 ArrayRef<FunctionSummary::ConstVCall>{});
|
|
|
454 Index.addGlobalValueSummary(Name, std::move(Summary));
|
|
|
455 } else {
|
|
|
456 std::unique_ptr<GlobalVarSummary> Summary =
|
|
|
457 llvm::make_unique<GlobalVarSummary>(GVFlags,
|
|
|
458 ArrayRef<ValueInfo>{});
|
|
|
459 Index.addGlobalValueSummary(Name, std::move(Summary));
|
|
|
460 }
|
|
|
461 });
|
|
120
|
462 }
|
|
|
463
|
|
|
464 // Compute summaries for all functions defined in module, and save in the
|
|
|
465 // index.
|
|
|
466 for (auto &F : M) {
|
|
|
467 if (F.isDeclaration())
|
|
|
468 continue;
|
|
|
469
|
|
|
470 BlockFrequencyInfo *BFI = nullptr;
|
|
|
471 std::unique_ptr<BlockFrequencyInfo> BFIPtr;
|
|
|
472 if (GetBFICallback)
|
|
|
473 BFI = GetBFICallback(F);
|
|
134
|
474 else if (F.hasProfileData()) {
|
|
120
|
475 LoopInfo LI{DominatorTree(const_cast<Function &>(F))};
|
|
|
476 BranchProbabilityInfo BPI{F, LI};
|
|
|
477 BFIPtr = llvm::make_unique<BlockFrequencyInfo>(F, BPI, LI);
|
|
|
478 BFI = BFIPtr.get();
|
|
|
479 }
|
|
|
480
|
|
121
|
481 computeFunctionSummary(Index, M, F, BFI, PSI,
|
|
|
482 !LocalsUsed.empty() || HasLocalInlineAsmSymbol,
|
|
|
483 CantBePromoted);
|
|
120
|
484 }
|
|
|
485
|
|
|
486 // Compute summaries for all variables defined in module, and save in the
|
|
|
487 // index.
|
|
|
488 for (const GlobalVariable &G : M.globals()) {
|
|
|
489 if (G.isDeclaration())
|
|
|
490 continue;
|
|
121
|
491 computeVariableSummary(Index, G, CantBePromoted);
|
|
120
|
492 }
|
|
|
493
|
|
|
494 // Compute summaries for all aliases defined in module, and save in the
|
|
|
495 // index.
|
|
|
496 for (const GlobalAlias &A : M.aliases())
|
|
121
|
497 computeAliasSummary(Index, A, CantBePromoted);
|
|
120
|
498
|
|
|
499 for (auto *V : LocalsUsed) {
|
|
|
500 auto *Summary = Index.getGlobalValueSummary(*V);
|
|
|
501 assert(Summary && "Missing summary for global value");
|
|
121
|
502 Summary->setNotEligibleToImport();
|
|
120
|
503 }
|
|
|
504
|
|
121
|
505 // The linker doesn't know about these LLVM produced values, so we need
|
|
|
506 // to flag them as live in the index to ensure index-based dead value
|
|
|
507 // analysis treats them as live roots of the analysis.
|
|
|
508 setLiveRoot(Index, "llvm.used");
|
|
|
509 setLiveRoot(Index, "llvm.compiler.used");
|
|
|
510 setLiveRoot(Index, "llvm.global_ctors");
|
|
|
511 setLiveRoot(Index, "llvm.global_dtors");
|
|
|
512 setLiveRoot(Index, "llvm.global.annotations");
|
|
|
513
|
|
|
514 bool IsThinLTO = true;
|
|
|
515 if (auto *MD =
|
|
|
516 mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("ThinLTO")))
|
|
|
517 IsThinLTO = MD->getZExtValue();
|
|
|
518
|
|
|
519 for (auto &GlobalList : Index) {
|
|
|
520 // Ignore entries for references that are undefined in the current module.
|
|
|
521 if (GlobalList.second.SummaryList.empty())
|
|
|
522 continue;
|
|
|
523
|
|
|
524 assert(GlobalList.second.SummaryList.size() == 1 &&
|
|
|
525 "Expected module's index to have one summary per GUID");
|
|
|
526 auto &Summary = GlobalList.second.SummaryList[0];
|
|
|
527 if (!IsThinLTO) {
|
|
|
528 Summary->setNotEligibleToImport();
|
|
|
529 continue;
|
|
|
530 }
|
|
|
531
|
|
|
532 bool AllRefsCanBeExternallyReferenced =
|
|
|
533 llvm::all_of(Summary->refs(), [&](const ValueInfo &VI) {
|
|
|
534 return !CantBePromoted.count(VI.getGUID());
|
|
120
|
535 });
|
|
121
|
536 if (!AllRefsCanBeExternallyReferenced) {
|
|
|
537 Summary->setNotEligibleToImport();
|
|
|
538 continue;
|
|
|
539 }
|
|
|
540
|
|
|
541 if (auto *FuncSummary = dyn_cast<FunctionSummary>(Summary.get())) {
|
|
|
542 bool AllCallsCanBeExternallyReferenced = llvm::all_of(
|
|
|
543 FuncSummary->calls(), [&](const FunctionSummary::EdgeTy &Edge) {
|
|
|
544 return !CantBePromoted.count(Edge.first.getGUID());
|
|
|
545 });
|
|
|
546 if (!AllCallsCanBeExternallyReferenced)
|
|
|
547 Summary->setNotEligibleToImport();
|
|
|
548 }
|
|
120
|
549 }
|
|
|
550
|
|
|
551 return Index;
|
|
|
552 }
|
|
|
553
|
|
|
554 AnalysisKey ModuleSummaryIndexAnalysis::Key;
|
|
|
555
|
|
|
556 ModuleSummaryIndex
|
|
|
557 ModuleSummaryIndexAnalysis::run(Module &M, ModuleAnalysisManager &AM) {
|
|
|
558 ProfileSummaryInfo &PSI = AM.getResult<ProfileSummaryAnalysis>(M);
|
|
|
559 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
|
|
|
560 return buildModuleSummaryIndex(
|
|
|
561 M,
|
|
|
562 [&FAM](const Function &F) {
|
|
|
563 return &FAM.getResult<BlockFrequencyAnalysis>(
|
|
|
564 *const_cast<Function *>(&F));
|
|
|
565 },
|
|
|
566 &PSI);
|
|
|
567 }
|
|
|
568
|
|
|
569 char ModuleSummaryIndexWrapperPass::ID = 0;
|
|
121
|
570
|
|
120
|
571 INITIALIZE_PASS_BEGIN(ModuleSummaryIndexWrapperPass, "module-summary-analysis",
|
|
|
572 "Module Summary Analysis", false, true)
|
|
|
573 INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
|
|
121
|
574 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
|
|
120
|
575 INITIALIZE_PASS_END(ModuleSummaryIndexWrapperPass, "module-summary-analysis",
|
|
|
576 "Module Summary Analysis", false, true)
|
|
|
577
|
|
|
578 ModulePass *llvm::createModuleSummaryIndexWrapperPass() {
|
|
|
579 return new ModuleSummaryIndexWrapperPass();
|
|
|
580 }
|
|
|
581
|
|
|
582 ModuleSummaryIndexWrapperPass::ModuleSummaryIndexWrapperPass()
|
|
|
583 : ModulePass(ID) {
|
|
|
584 initializeModuleSummaryIndexWrapperPassPass(*PassRegistry::getPassRegistry());
|
|
|
585 }
|
|
|
586
|
|
|
587 bool ModuleSummaryIndexWrapperPass::runOnModule(Module &M) {
|
|
|
588 auto &PSI = *getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
|
|
|
589 Index = buildModuleSummaryIndex(
|
|
|
590 M,
|
|
|
591 [this](const Function &F) {
|
|
|
592 return &(this->getAnalysis<BlockFrequencyInfoWrapperPass>(
|
|
|
593 *const_cast<Function *>(&F))
|
|
|
594 .getBFI());
|
|
|
595 },
|
|
|
596 &PSI);
|
|
|
597 return false;
|
|
|
598 }
|
|
|
599
|
|
|
600 bool ModuleSummaryIndexWrapperPass::doFinalization(Module &M) {
|
|
|
601 Index.reset();
|
|
|
602 return false;
|
|
|
603 }
|
|
|
604
|
|
|
605 void ModuleSummaryIndexWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
|
|
|
606 AU.setPreservesAll();
|
|
|
607 AU.addRequired<BlockFrequencyInfoWrapperPass>();
|
|
|
608 AU.addRequired<ProfileSummaryInfoWrapperPass>();
|
|
|
609 }
|
