|
150
|
1 //===--- QueryDriverDatabase.cpp ---------------------------------*- C++-*-===//
|
|
|
2 //
|
|
|
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
4 // See https://llvm.org/LICENSE.txt for license information.
|
|
|
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
6 //
|
|
|
7 //===----------------------------------------------------------------------===//
|
|
|
8 // Some compiler drivers have implicit search mechanism for system headers.
|
|
|
9 // This compilation database implementation tries to extract that information by
|
|
|
10 // executing the driver in verbose mode. gcc-compatible drivers print something
|
|
|
11 // like:
|
|
|
12 // ....
|
|
|
13 // ....
|
|
|
14 // #include <...> search starts here:
|
|
|
15 // /usr/lib/gcc/x86_64-linux-gnu/7/include
|
|
|
16 // /usr/local/include
|
|
|
17 // /usr/lib/gcc/x86_64-linux-gnu/7/include-fixed
|
|
|
18 // /usr/include/x86_64-linux-gnu
|
|
|
19 // /usr/include
|
|
|
20 // End of search list.
|
|
|
21 // ....
|
|
|
22 // ....
|
|
|
23 // This component parses that output and adds each path to command line args
|
|
|
24 // provided by Base, after prepending them with -isystem. Therefore current
|
|
|
25 // implementation would not work with a driver that is not gcc-compatible.
|
|
|
26 //
|
|
|
27 // First argument of the command line received from underlying compilation
|
|
|
28 // database is used as compiler driver path. Due to this arbitrary binary
|
|
|
29 // execution, this mechanism is not used by default and only executes binaries
|
|
|
30 // in the paths that are explicitly whitelisted by the user.
|
|
|
31
|
|
|
32 #include "GlobalCompilationDatabase.h"
|
|
173
|
33 #include "support/Logger.h"
|
|
|
34 #include "support/Path.h"
|
|
|
35 #include "support/Trace.h"
|
|
150
|
36 #include "clang/Driver/Types.h"
|
|
|
37 #include "clang/Tooling/CompilationDatabase.h"
|
|
|
38 #include "llvm/ADT/DenseMap.h"
|
|
|
39 #include "llvm/ADT/ScopeExit.h"
|
|
|
40 #include "llvm/ADT/SmallString.h"
|
|
|
41 #include "llvm/ADT/StringExtras.h"
|
|
|
42 #include "llvm/ADT/StringRef.h"
|
|
|
43 #include "llvm/ADT/iterator_range.h"
|
|
|
44 #include "llvm/Support/FileSystem.h"
|
|
|
45 #include "llvm/Support/MemoryBuffer.h"
|
|
|
46 #include "llvm/Support/Path.h"
|
|
|
47 #include "llvm/Support/Program.h"
|
|
|
48 #include "llvm/Support/Regex.h"
|
|
|
49 #include "llvm/Support/ScopedPrinter.h"
|
|
|
50 #include <algorithm>
|
|
|
51 #include <map>
|
|
|
52 #include <string>
|
|
|
53 #include <vector>
|
|
|
54
|
|
|
55 namespace clang {
|
|
|
56 namespace clangd {
|
|
|
57 namespace {
|
|
|
58
|
|
|
59 std::vector<std::string> parseDriverOutput(llvm::StringRef Output) {
|
|
|
60 std::vector<std::string> SystemIncludes;
|
|
|
61 const char SIS[] = "#include <...> search starts here:";
|
|
|
62 const char SIE[] = "End of search list.";
|
|
|
63 llvm::SmallVector<llvm::StringRef, 8> Lines;
|
|
|
64 Output.split(Lines, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
|
|
|
65
|
|
|
66 auto StartIt = llvm::find_if(
|
|
|
67 Lines, [SIS](llvm::StringRef Line) { return Line.trim() == SIS; });
|
|
|
68 if (StartIt == Lines.end()) {
|
|
|
69 elog("System include extraction: start marker not found: {0}", Output);
|
|
|
70 return {};
|
|
|
71 }
|
|
|
72 ++StartIt;
|
|
|
73 const auto EndIt =
|
|
|
74 llvm::find_if(llvm::make_range(StartIt, Lines.end()),
|
|
|
75 [SIE](llvm::StringRef Line) { return Line.trim() == SIE; });
|
|
|
76 if (EndIt == Lines.end()) {
|
|
|
77 elog("System include extraction: end marker missing: {0}", Output);
|
|
|
78 return {};
|
|
|
79 }
|
|
|
80
|
|
|
81 for (llvm::StringRef Line : llvm::make_range(StartIt, EndIt)) {
|
|
|
82 SystemIncludes.push_back(Line.trim().str());
|
|
|
83 vlog("System include extraction: adding {0}", Line);
|
|
|
84 }
|
|
|
85 return SystemIncludes;
|
|
|
86 }
|
|
|
87
|
|
|
88 std::vector<std::string>
|
|
|
89 extractSystemIncludes(PathRef Driver, llvm::StringRef Lang,
|
|
|
90 llvm::ArrayRef<std::string> CommandLine,
|
|
|
91 llvm::Regex &QueryDriverRegex) {
|
|
|
92 trace::Span Tracer("Extract system includes");
|
|
|
93 SPAN_ATTACH(Tracer, "driver", Driver);
|
|
|
94 SPAN_ATTACH(Tracer, "lang", Lang);
|
|
|
95
|
|
|
96 if (!QueryDriverRegex.match(Driver)) {
|
|
|
97 vlog("System include extraction: not whitelisted driver {0}", Driver);
|
|
|
98 return {};
|
|
|
99 }
|
|
|
100
|
|
|
101 if (!llvm::sys::fs::exists(Driver)) {
|
|
|
102 elog("System include extraction: {0} does not exist.", Driver);
|
|
|
103 return {};
|
|
|
104 }
|
|
|
105 if (!llvm::sys::fs::can_execute(Driver)) {
|
|
|
106 elog("System include extraction: {0} is not executable.", Driver);
|
|
|
107 return {};
|
|
|
108 }
|
|
|
109
|
|
|
110 llvm::SmallString<128> StdErrPath;
|
|
|
111 if (auto EC = llvm::sys::fs::createTemporaryFile("system-includes", "clangd",
|
|
|
112 StdErrPath)) {
|
|
|
113 elog("System include extraction: failed to create temporary file with "
|
|
|
114 "error {0}",
|
|
|
115 EC.message());
|
|
|
116 return {};
|
|
|
117 }
|
|
|
118 auto CleanUp = llvm::make_scope_exit(
|
|
|
119 [&StdErrPath]() { llvm::sys::fs::remove(StdErrPath); });
|
|
|
120
|
|
|
121 llvm::Optional<llvm::StringRef> Redirects[] = {
|
|
|
122 {""}, {""}, llvm::StringRef(StdErrPath)};
|
|
|
123
|
|
|
124 llvm::SmallVector<llvm::StringRef, 12> Args = {Driver, "-E", "-x",
|
|
|
125 Lang, "-", "-v"};
|
|
|
126
|
|
|
127 // These flags will be preserved
|
|
|
128 const llvm::StringRef FlagsToPreserve[] = {
|
|
|
129 "-nostdinc", "--no-standard-includes", "-nostdinc++", "-nobuiltininc"};
|
|
|
130 // Preserves these flags and their values, either as separate args or with an
|
|
|
131 // equalsbetween them
|
|
|
132 const llvm::StringRef ArgsToPreserve[] = {"--sysroot", "-isysroot"};
|
|
|
133
|
|
|
134 for (size_t I = 0, E = CommandLine.size(); I < E; ++I) {
|
|
|
135 llvm::StringRef Arg = CommandLine[I];
|
|
|
136 if (llvm::any_of(FlagsToPreserve,
|
|
|
137 [&Arg](llvm::StringRef S) { return S == Arg; })) {
|
|
|
138 Args.push_back(Arg);
|
|
|
139 } else {
|
|
|
140 const auto *Found =
|
|
|
141 llvm::find_if(ArgsToPreserve, [&Arg](llvm::StringRef S) {
|
|
|
142 return Arg.startswith(S);
|
|
|
143 });
|
|
|
144 if (Found == std::end(ArgsToPreserve))
|
|
|
145 continue;
|
|
|
146 Arg.consume_front(*Found);
|
|
|
147 if (Arg.empty() && I + 1 < E) {
|
|
|
148 Args.push_back(CommandLine[I]);
|
|
|
149 Args.push_back(CommandLine[++I]);
|
|
|
150 } else if (Arg.startswith("=")) {
|
|
|
151 Args.push_back(CommandLine[I]);
|
|
|
152 }
|
|
|
153 }
|
|
|
154 }
|
|
|
155
|
|
|
156 if (int RC = llvm::sys::ExecuteAndWait(Driver, Args, /*Env=*/llvm::None,
|
|
|
157 Redirects)) {
|
|
|
158 elog("System include extraction: driver execution failed with return code: "
|
|
|
159 "{0}. Args: ['{1}']",
|
|
|
160 llvm::to_string(RC), llvm::join(Args, "', '"));
|
|
|
161 return {};
|
|
|
162 }
|
|
|
163
|
|
|
164 auto BufOrError = llvm::MemoryBuffer::getFile(StdErrPath);
|
|
|
165 if (!BufOrError) {
|
|
|
166 elog("System include extraction: failed to read {0} with error {1}",
|
|
|
167 StdErrPath, BufOrError.getError().message());
|
|
|
168 return {};
|
|
|
169 }
|
|
|
170
|
|
|
171 auto Includes = parseDriverOutput(BufOrError->get()->getBuffer());
|
|
173
|
172 log("System include extractor: successfully executed {0}, got includes: "
|
|
150
|
173 "\"{1}\"",
|
|
|
174 Driver, llvm::join(Includes, ", "));
|
|
|
175 return Includes;
|
|
|
176 }
|
|
|
177
|
|
|
178 tooling::CompileCommand &
|
|
|
179 addSystemIncludes(tooling::CompileCommand &Cmd,
|
|
|
180 llvm::ArrayRef<std::string> SystemIncludes) {
|
|
|
181 for (llvm::StringRef Include : SystemIncludes) {
|
|
|
182 // FIXME(kadircet): This doesn't work when we have "--driver-mode=cl"
|
|
|
183 Cmd.CommandLine.push_back("-isystem");
|
|
|
184 Cmd.CommandLine.push_back(Include.str());
|
|
|
185 }
|
|
|
186 return Cmd;
|
|
|
187 }
|
|
|
188
|
|
|
189 /// Converts a glob containing only ** or * into a regex.
|
|
|
190 std::string convertGlobToRegex(llvm::StringRef Glob) {
|
|
|
191 std::string RegText;
|
|
|
192 llvm::raw_string_ostream RegStream(RegText);
|
|
|
193 RegStream << '^';
|
|
|
194 for (size_t I = 0, E = Glob.size(); I < E; ++I) {
|
|
|
195 if (Glob[I] == '*') {
|
|
|
196 if (I + 1 < E && Glob[I + 1] == '*') {
|
|
|
197 // Double star, accept any sequence.
|
|
|
198 RegStream << ".*";
|
|
|
199 // Also skip the second star.
|
|
|
200 ++I;
|
|
|
201 } else {
|
|
|
202 // Single star, accept any sequence without a slash.
|
|
|
203 RegStream << "[^/]*";
|
|
|
204 }
|
|
|
205 } else {
|
|
|
206 RegStream << llvm::Regex::escape(Glob.substr(I, 1));
|
|
|
207 }
|
|
|
208 }
|
|
|
209 RegStream << '$';
|
|
|
210 RegStream.flush();
|
|
|
211 return RegText;
|
|
|
212 }
|
|
|
213
|
|
|
214 /// Converts a glob containing only ** or * into a regex.
|
|
|
215 llvm::Regex convertGlobsToRegex(llvm::ArrayRef<std::string> Globs) {
|
|
|
216 assert(!Globs.empty() && "Globs cannot be empty!");
|
|
|
217 std::vector<std::string> RegTexts;
|
|
|
218 RegTexts.reserve(Globs.size());
|
|
|
219 for (llvm::StringRef Glob : Globs)
|
|
|
220 RegTexts.push_back(convertGlobToRegex(Glob));
|
|
|
221
|
|
|
222 llvm::Regex Reg(llvm::join(RegTexts, "|"));
|
|
|
223 assert(Reg.isValid(RegTexts.front()) &&
|
|
|
224 "Created an invalid regex from globs");
|
|
|
225 return Reg;
|
|
|
226 }
|
|
|
227
|
|
|
228 /// Extracts system includes from a trusted driver by parsing the output of
|
|
|
229 /// include search path and appends them to the commands coming from underlying
|
|
|
230 /// compilation database.
|
|
|
231 class QueryDriverDatabase : public GlobalCompilationDatabase {
|
|
|
232 public:
|
|
|
233 QueryDriverDatabase(llvm::ArrayRef<std::string> QueryDriverGlobs,
|
|
|
234 std::unique_ptr<GlobalCompilationDatabase> Base)
|
|
|
235 : QueryDriverRegex(convertGlobsToRegex(QueryDriverGlobs)),
|
|
|
236 Base(std::move(Base)) {
|
|
|
237 assert(this->Base);
|
|
|
238 BaseChanged =
|
|
|
239 this->Base->watch([this](const std::vector<std::string> &Changes) {
|
|
|
240 OnCommandChanged.broadcast(Changes);
|
|
|
241 });
|
|
|
242 }
|
|
|
243
|
|
|
244 llvm::Optional<tooling::CompileCommand>
|
|
|
245 getCompileCommand(PathRef File) const override {
|
|
|
246 auto Cmd = Base->getCompileCommand(File);
|
|
|
247 if (!Cmd || Cmd->CommandLine.empty())
|
|
|
248 return Cmd;
|
|
|
249
|
|
|
250 llvm::StringRef Lang;
|
|
|
251 for (size_t I = 0, E = Cmd->CommandLine.size(); I < E; ++I) {
|
|
|
252 llvm::StringRef Arg = Cmd->CommandLine[I];
|
|
|
253 if (Arg == "-x" && I + 1 < E)
|
|
|
254 Lang = Cmd->CommandLine[I + 1];
|
|
|
255 else if (Arg.startswith("-x"))
|
|
|
256 Lang = Arg.drop_front(2).trim();
|
|
|
257 }
|
|
|
258 if (Lang.empty()) {
|
|
|
259 llvm::StringRef Ext = llvm::sys::path::extension(File).trim('.');
|
|
|
260 auto Type = driver::types::lookupTypeForExtension(Ext);
|
|
|
261 if (Type == driver::types::TY_INVALID) {
|
|
|
262 elog("System include extraction: invalid file type for {0}", Ext);
|
|
|
263 return {};
|
|
|
264 }
|
|
|
265 Lang = driver::types::getTypeName(Type);
|
|
|
266 }
|
|
|
267
|
|
|
268 llvm::SmallString<128> Driver(Cmd->CommandLine.front());
|
|
|
269 llvm::sys::fs::make_absolute(Cmd->Directory, Driver);
|
|
|
270 auto Key = std::make_pair(Driver.str().str(), Lang.str());
|
|
|
271
|
|
|
272 std::vector<std::string> SystemIncludes;
|
|
|
273 {
|
|
|
274 std::lock_guard<std::mutex> Lock(Mu);
|
|
|
275
|
|
|
276 auto It = DriverToIncludesCache.find(Key);
|
|
|
277 if (It != DriverToIncludesCache.end())
|
|
|
278 SystemIncludes = It->second;
|
|
|
279 else
|
|
|
280 DriverToIncludesCache[Key] = SystemIncludes = extractSystemIncludes(
|
|
|
281 Key.first, Key.second, Cmd->CommandLine, QueryDriverRegex);
|
|
|
282 }
|
|
|
283
|
|
|
284 return addSystemIncludes(*Cmd, SystemIncludes);
|
|
|
285 }
|
|
|
286
|
|
|
287 llvm::Optional<ProjectInfo> getProjectInfo(PathRef File) const override {
|
|
|
288 return Base->getProjectInfo(File);
|
|
|
289 }
|
|
|
290
|
|
|
291 private:
|
|
|
292 mutable std::mutex Mu;
|
|
|
293 // Caches includes extracted from a driver.
|
|
|
294 mutable std::map<std::pair<std::string, std::string>,
|
|
|
295 std::vector<std::string>>
|
|
|
296 DriverToIncludesCache;
|
|
|
297 mutable llvm::Regex QueryDriverRegex;
|
|
|
298
|
|
|
299 std::unique_ptr<GlobalCompilationDatabase> Base;
|
|
|
300 CommandChanged::Subscription BaseChanged;
|
|
|
301 };
|
|
|
302 } // namespace
|
|
|
303
|
|
|
304 std::unique_ptr<GlobalCompilationDatabase>
|
|
|
305 getQueryDriverDatabase(llvm::ArrayRef<std::string> QueryDriverGlobs,
|
|
|
306 std::unique_ptr<GlobalCompilationDatabase> Base) {
|
|
|
307 assert(Base && "Null base to SystemIncludeExtractor");
|
|
|
308 if (QueryDriverGlobs.empty())
|
|
|
309 return Base;
|
|
|
310 return std::make_unique<QueryDriverDatabase>(QueryDriverGlobs,
|
|
|
311 std::move(Base));
|
|
|
312 }
|
|
|
313
|
|
|
314 } // namespace clangd
|
|
|
315 } // namespace clang
|
