|
150
|
1 //===- JSONCompilationDatabase.cpp ----------------------------------------===//
|
|
|
2 //
|
|
|
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
4 // See https://llvm.org/LICENSE.txt for license information.
|
|
|
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
6 //
|
|
|
7 //===----------------------------------------------------------------------===//
|
|
|
8 //
|
|
|
9 // This file contains the implementation of the JSONCompilationDatabase.
|
|
|
10 //
|
|
|
11 //===----------------------------------------------------------------------===//
|
|
|
12
|
|
|
13 #include "clang/Tooling/JSONCompilationDatabase.h"
|
|
|
14 #include "clang/Basic/LLVM.h"
|
|
|
15 #include "clang/Tooling/CompilationDatabase.h"
|
|
|
16 #include "clang/Tooling/CompilationDatabasePluginRegistry.h"
|
|
|
17 #include "clang/Tooling/Tooling.h"
|
|
|
18 #include "llvm/ADT/Optional.h"
|
|
|
19 #include "llvm/ADT/STLExtras.h"
|
|
|
20 #include "llvm/ADT/SmallString.h"
|
|
|
21 #include "llvm/ADT/SmallVector.h"
|
|
|
22 #include "llvm/ADT/StringRef.h"
|
|
|
23 #include "llvm/ADT/Triple.h"
|
|
|
24 #include "llvm/Support/Allocator.h"
|
|
|
25 #include "llvm/Support/Casting.h"
|
|
|
26 #include "llvm/Support/CommandLine.h"
|
|
|
27 #include "llvm/Support/ErrorOr.h"
|
|
|
28 #include "llvm/Support/Host.h"
|
|
|
29 #include "llvm/Support/MemoryBuffer.h"
|
|
|
30 #include "llvm/Support/Path.h"
|
|
|
31 #include "llvm/Support/StringSaver.h"
|
|
|
32 #include "llvm/Support/VirtualFileSystem.h"
|
|
|
33 #include "llvm/Support/YAMLParser.h"
|
|
|
34 #include "llvm/Support/raw_ostream.h"
|
|
|
35 #include <cassert>
|
|
|
36 #include <memory>
|
|
|
37 #include <string>
|
|
|
38 #include <system_error>
|
|
|
39 #include <tuple>
|
|
|
40 #include <utility>
|
|
|
41 #include <vector>
|
|
|
42
|
|
|
43 using namespace clang;
|
|
|
44 using namespace tooling;
|
|
|
45
|
|
|
46 namespace {
|
|
|
47
|
|
|
48 /// A parser for escaped strings of command line arguments.
|
|
|
49 ///
|
|
|
50 /// Assumes \-escaping for quoted arguments (see the documentation of
|
|
|
51 /// unescapeCommandLine(...)).
|
|
|
52 class CommandLineArgumentParser {
|
|
|
53 public:
|
|
|
54 CommandLineArgumentParser(StringRef CommandLine)
|
|
|
55 : Input(CommandLine), Position(Input.begin()-1) {}
|
|
|
56
|
|
|
57 std::vector<std::string> parse() {
|
|
|
58 bool HasMoreInput = true;
|
|
|
59 while (HasMoreInput && nextNonWhitespace()) {
|
|
|
60 std::string Argument;
|
|
|
61 HasMoreInput = parseStringInto(Argument);
|
|
|
62 CommandLine.push_back(Argument);
|
|
|
63 }
|
|
|
64 return CommandLine;
|
|
|
65 }
|
|
|
66
|
|
|
67 private:
|
|
|
68 // All private methods return true if there is more input available.
|
|
|
69
|
|
|
70 bool parseStringInto(std::string &String) {
|
|
|
71 do {
|
|
|
72 if (*Position == '"') {
|
|
|
73 if (!parseDoubleQuotedStringInto(String)) return false;
|
|
|
74 } else if (*Position == '\'') {
|
|
|
75 if (!parseSingleQuotedStringInto(String)) return false;
|
|
|
76 } else {
|
|
|
77 if (!parseFreeStringInto(String)) return false;
|
|
|
78 }
|
|
|
79 } while (*Position != ' ');
|
|
|
80 return true;
|
|
|
81 }
|
|
|
82
|
|
|
83 bool parseDoubleQuotedStringInto(std::string &String) {
|
|
|
84 if (!next()) return false;
|
|
|
85 while (*Position != '"') {
|
|
|
86 if (!skipEscapeCharacter()) return false;
|
|
|
87 String.push_back(*Position);
|
|
|
88 if (!next()) return false;
|
|
|
89 }
|
|
|
90 return next();
|
|
|
91 }
|
|
|
92
|
|
|
93 bool parseSingleQuotedStringInto(std::string &String) {
|
|
|
94 if (!next()) return false;
|
|
|
95 while (*Position != '\'') {
|
|
|
96 String.push_back(*Position);
|
|
|
97 if (!next()) return false;
|
|
|
98 }
|
|
|
99 return next();
|
|
|
100 }
|
|
|
101
|
|
|
102 bool parseFreeStringInto(std::string &String) {
|
|
|
103 do {
|
|
|
104 if (!skipEscapeCharacter()) return false;
|
|
|
105 String.push_back(*Position);
|
|
|
106 if (!next()) return false;
|
|
|
107 } while (*Position != ' ' && *Position != '"' && *Position != '\'');
|
|
|
108 return true;
|
|
|
109 }
|
|
|
110
|
|
|
111 bool skipEscapeCharacter() {
|
|
|
112 if (*Position == '\\') {
|
|
|
113 return next();
|
|
|
114 }
|
|
|
115 return true;
|
|
|
116 }
|
|
|
117
|
|
|
118 bool nextNonWhitespace() {
|
|
|
119 do {
|
|
|
120 if (!next()) return false;
|
|
|
121 } while (*Position == ' ');
|
|
|
122 return true;
|
|
|
123 }
|
|
|
124
|
|
|
125 bool next() {
|
|
|
126 ++Position;
|
|
|
127 return Position != Input.end();
|
|
|
128 }
|
|
|
129
|
|
|
130 const StringRef Input;
|
|
|
131 StringRef::iterator Position;
|
|
|
132 std::vector<std::string> CommandLine;
|
|
|
133 };
|
|
|
134
|
|
|
135 std::vector<std::string> unescapeCommandLine(JSONCommandLineSyntax Syntax,
|
|
|
136 StringRef EscapedCommandLine) {
|
|
|
137 if (Syntax == JSONCommandLineSyntax::AutoDetect) {
|
|
|
138 Syntax = JSONCommandLineSyntax::Gnu;
|
|
|
139 llvm::Triple Triple(llvm::sys::getProcessTriple());
|
|
|
140 if (Triple.getOS() == llvm::Triple::OSType::Win32) {
|
|
|
141 // Assume Windows command line parsing on Win32 unless the triple
|
|
|
142 // explicitly tells us otherwise.
|
|
|
143 if (!Triple.hasEnvironment() ||
|
|
|
144 Triple.getEnvironment() == llvm::Triple::EnvironmentType::MSVC)
|
|
|
145 Syntax = JSONCommandLineSyntax::Windows;
|
|
|
146 }
|
|
|
147 }
|
|
|
148
|
|
|
149 if (Syntax == JSONCommandLineSyntax::Windows) {
|
|
|
150 llvm::BumpPtrAllocator Alloc;
|
|
|
151 llvm::StringSaver Saver(Alloc);
|
|
|
152 llvm::SmallVector<const char *, 64> T;
|
|
|
153 llvm::cl::TokenizeWindowsCommandLine(EscapedCommandLine, Saver, T);
|
|
|
154 std::vector<std::string> Result(T.begin(), T.end());
|
|
|
155 return Result;
|
|
|
156 }
|
|
|
157 assert(Syntax == JSONCommandLineSyntax::Gnu);
|
|
|
158 CommandLineArgumentParser parser(EscapedCommandLine);
|
|
|
159 return parser.parse();
|
|
|
160 }
|
|
|
161
|
|
|
162 // This plugin locates a nearby compile_command.json file, and also infers
|
|
|
163 // compile commands for files not present in the database.
|
|
|
164 class JSONCompilationDatabasePlugin : public CompilationDatabasePlugin {
|
|
|
165 std::unique_ptr<CompilationDatabase>
|
|
|
166 loadFromDirectory(StringRef Directory, std::string &ErrorMessage) override {
|
|
|
167 SmallString<1024> JSONDatabasePath(Directory);
|
|
|
168 llvm::sys::path::append(JSONDatabasePath, "compile_commands.json");
|
|
|
169 auto Base = JSONCompilationDatabase::loadFromFile(
|
|
|
170 JSONDatabasePath, ErrorMessage, JSONCommandLineSyntax::AutoDetect);
|
|
|
171 return Base ? inferTargetAndDriverMode(
|
|
|
172 inferMissingCompileCommands(expandResponseFiles(
|
|
|
173 std::move(Base), llvm::vfs::getRealFileSystem())))
|
|
|
174 : nullptr;
|
|
|
175 }
|
|
|
176 };
|
|
|
177
|
|
|
178 } // namespace
|
|
|
179
|
|
|
180 // Register the JSONCompilationDatabasePlugin with the
|
|
|
181 // CompilationDatabasePluginRegistry using this statically initialized variable.
|
|
|
182 static CompilationDatabasePluginRegistry::Add<JSONCompilationDatabasePlugin>
|
|
|
183 X("json-compilation-database", "Reads JSON formatted compilation databases");
|
|
|
184
|
|
|
185 namespace clang {
|
|
|
186 namespace tooling {
|
|
|
187
|
|
|
188 // This anchor is used to force the linker to link in the generated object file
|
|
|
189 // and thus register the JSONCompilationDatabasePlugin.
|
|
|
190 volatile int JSONAnchorSource = 0;
|
|
|
191
|
|
|
192 } // namespace tooling
|
|
|
193 } // namespace clang
|
|
|
194
|
|
|
195 std::unique_ptr<JSONCompilationDatabase>
|
|
|
196 JSONCompilationDatabase::loadFromFile(StringRef FilePath,
|
|
|
197 std::string &ErrorMessage,
|
|
|
198 JSONCommandLineSyntax Syntax) {
|
|
|
199 // Don't mmap: if we're a long-lived process, the build system may overwrite.
|
|
|
200 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> DatabaseBuffer =
|
|
|
201 llvm::MemoryBuffer::getFile(FilePath, /*FileSize=*/-1,
|
|
|
202 /*RequiresNullTerminator=*/true,
|
|
|
203 /*IsVolatile=*/true);
|
|
|
204 if (std::error_code Result = DatabaseBuffer.getError()) {
|
|
|
205 ErrorMessage = "Error while opening JSON database: " + Result.message();
|
|
|
206 return nullptr;
|
|
|
207 }
|
|
|
208 std::unique_ptr<JSONCompilationDatabase> Database(
|
|
|
209 new JSONCompilationDatabase(std::move(*DatabaseBuffer), Syntax));
|
|
|
210 if (!Database->parse(ErrorMessage))
|
|
|
211 return nullptr;
|
|
|
212 return Database;
|
|
|
213 }
|
|
|
214
|
|
|
215 std::unique_ptr<JSONCompilationDatabase>
|
|
|
216 JSONCompilationDatabase::loadFromBuffer(StringRef DatabaseString,
|
|
|
217 std::string &ErrorMessage,
|
|
|
218 JSONCommandLineSyntax Syntax) {
|
|
|
219 std::unique_ptr<llvm::MemoryBuffer> DatabaseBuffer(
|
|
|
220 llvm::MemoryBuffer::getMemBuffer(DatabaseString));
|
|
|
221 std::unique_ptr<JSONCompilationDatabase> Database(
|
|
|
222 new JSONCompilationDatabase(std::move(DatabaseBuffer), Syntax));
|
|
|
223 if (!Database->parse(ErrorMessage))
|
|
|
224 return nullptr;
|
|
|
225 return Database;
|
|
|
226 }
|
|
|
227
|
|
|
228 std::vector<CompileCommand>
|
|
|
229 JSONCompilationDatabase::getCompileCommands(StringRef FilePath) const {
|
|
|
230 SmallString<128> NativeFilePath;
|
|
|
231 llvm::sys::path::native(FilePath, NativeFilePath);
|
|
|
232
|
|
|
233 std::string Error;
|
|
|
234 llvm::raw_string_ostream ES(Error);
|
|
|
235 StringRef Match = MatchTrie.findEquivalent(NativeFilePath, ES);
|
|
|
236 if (Match.empty())
|
|
|
237 return {};
|
|
|
238 const auto CommandsRefI = IndexByFile.find(Match);
|
|
|
239 if (CommandsRefI == IndexByFile.end())
|
|
|
240 return {};
|
|
|
241 std::vector<CompileCommand> Commands;
|
|
|
242 getCommands(CommandsRefI->getValue(), Commands);
|
|
|
243 return Commands;
|
|
|
244 }
|
|
|
245
|
|
|
246 std::vector<std::string>
|
|
|
247 JSONCompilationDatabase::getAllFiles() const {
|
|
|
248 std::vector<std::string> Result;
|
|
|
249 for (const auto &CommandRef : IndexByFile)
|
|
|
250 Result.push_back(CommandRef.first().str());
|
|
|
251 return Result;
|
|
|
252 }
|
|
|
253
|
|
|
254 std::vector<CompileCommand>
|
|
|
255 JSONCompilationDatabase::getAllCompileCommands() const {
|
|
|
256 std::vector<CompileCommand> Commands;
|
|
|
257 getCommands(AllCommands, Commands);
|
|
|
258 return Commands;
|
|
|
259 }
|
|
|
260
|
|
|
261 static llvm::StringRef stripExecutableExtension(llvm::StringRef Name) {
|
|
|
262 Name.consume_back(".exe");
|
|
|
263 return Name;
|
|
|
264 }
|
|
|
265
|
|
|
266 // There are compiler-wrappers (ccache, distcc, gomacc) that take the "real"
|
|
|
267 // compiler as an argument, e.g. distcc gcc -O3 foo.c.
|
|
|
268 // These end up in compile_commands.json when people set CC="distcc gcc".
|
|
|
269 // Clang's driver doesn't understand this, so we need to unwrap.
|
|
|
270 static bool unwrapCommand(std::vector<std::string> &Args) {
|
|
|
271 if (Args.size() < 2)
|
|
|
272 return false;
|
|
|
273 StringRef Wrapper =
|
|
|
274 stripExecutableExtension(llvm::sys::path::filename(Args.front()));
|
|
|
275 if (Wrapper == "distcc" || Wrapper == "gomacc" || Wrapper == "ccache") {
|
|
|
276 // Most of these wrappers support being invoked 3 ways:
|
|
|
277 // `distcc g++ file.c` This is the mode we're trying to match.
|
|
|
278 // We need to drop `distcc`.
|
|
|
279 // `distcc file.c` This acts like compiler is cc or similar.
|
|
|
280 // Clang's driver can handle this, no change needed.
|
|
|
281 // `g++ file.c` g++ is a symlink to distcc.
|
|
|
282 // We don't even notice this case, and all is well.
|
|
|
283 //
|
|
|
284 // We need to distinguish between the first and second case.
|
|
|
285 // The wrappers themselves don't take flags, so Args[1] is a compiler flag,
|
|
|
286 // an input file, or a compiler. Inputs have extensions, compilers don't.
|
|
|
287 bool HasCompiler =
|
|
|
288 (Args[1][0] != '-') &&
|
|
|
289 !llvm::sys::path::has_extension(stripExecutableExtension(Args[1]));
|
|
|
290 if (HasCompiler) {
|
|
|
291 Args.erase(Args.begin());
|
|
|
292 return true;
|
|
|
293 }
|
|
|
294 // If !HasCompiler, wrappers act like GCC. Fine: so do we.
|
|
|
295 }
|
|
|
296 return false;
|
|
|
297 }
|
|
|
298
|
|
|
299 static std::vector<std::string>
|
|
|
300 nodeToCommandLine(JSONCommandLineSyntax Syntax,
|
|
|
301 const std::vector<llvm::yaml::ScalarNode *> &Nodes) {
|
|
|
302 SmallString<1024> Storage;
|
|
|
303 std::vector<std::string> Arguments;
|
|
|
304 if (Nodes.size() == 1)
|
|
|
305 Arguments = unescapeCommandLine(Syntax, Nodes[0]->getValue(Storage));
|
|
|
306 else
|
|
|
307 for (const auto *Node : Nodes)
|
|
|
308 Arguments.push_back(std::string(Node->getValue(Storage)));
|
|
|
309 // There may be multiple wrappers: using distcc and ccache together is common.
|
|
|
310 while (unwrapCommand(Arguments))
|
|
|
311 ;
|
|
|
312 return Arguments;
|
|
|
313 }
|
|
|
314
|
|
|
315 void JSONCompilationDatabase::getCommands(
|
|
|
316 ArrayRef<CompileCommandRef> CommandsRef,
|
|
|
317 std::vector<CompileCommand> &Commands) const {
|
|
|
318 for (const auto &CommandRef : CommandsRef) {
|
|
|
319 SmallString<8> DirectoryStorage;
|
|
|
320 SmallString<32> FilenameStorage;
|
|
|
321 SmallString<32> OutputStorage;
|
|
|
322 auto Output = std::get<3>(CommandRef);
|
|
|
323 Commands.emplace_back(
|
|
|
324 std::get<0>(CommandRef)->getValue(DirectoryStorage),
|
|
|
325 std::get<1>(CommandRef)->getValue(FilenameStorage),
|
|
|
326 nodeToCommandLine(Syntax, std::get<2>(CommandRef)),
|
|
|
327 Output ? Output->getValue(OutputStorage) : "");
|
|
|
328 }
|
|
|
329 }
|
|
|
330
|
|
|
331 bool JSONCompilationDatabase::parse(std::string &ErrorMessage) {
|
|
|
332 llvm::yaml::document_iterator I = YAMLStream.begin();
|
|
|
333 if (I == YAMLStream.end()) {
|
|
|
334 ErrorMessage = "Error while parsing YAML.";
|
|
|
335 return false;
|
|
|
336 }
|
|
|
337 llvm::yaml::Node *Root = I->getRoot();
|
|
|
338 if (!Root) {
|
|
|
339 ErrorMessage = "Error while parsing YAML.";
|
|
|
340 return false;
|
|
|
341 }
|
|
|
342 auto *Array = dyn_cast<llvm::yaml::SequenceNode>(Root);
|
|
|
343 if (!Array) {
|
|
|
344 ErrorMessage = "Expected array.";
|
|
|
345 return false;
|
|
|
346 }
|
|
|
347 for (auto &NextObject : *Array) {
|
|
|
348 auto *Object = dyn_cast<llvm::yaml::MappingNode>(&NextObject);
|
|
|
349 if (!Object) {
|
|
|
350 ErrorMessage = "Expected object.";
|
|
|
351 return false;
|
|
|
352 }
|
|
|
353 llvm::yaml::ScalarNode *Directory = nullptr;
|
|
|
354 llvm::Optional<std::vector<llvm::yaml::ScalarNode *>> Command;
|
|
|
355 llvm::yaml::ScalarNode *File = nullptr;
|
|
|
356 llvm::yaml::ScalarNode *Output = nullptr;
|
|
|
357 for (auto& NextKeyValue : *Object) {
|
|
|
358 auto *KeyString = dyn_cast<llvm::yaml::ScalarNode>(NextKeyValue.getKey());
|
|
|
359 if (!KeyString) {
|
|
|
360 ErrorMessage = "Expected strings as key.";
|
|
|
361 return false;
|
|
|
362 }
|
|
|
363 SmallString<10> KeyStorage;
|
|
|
364 StringRef KeyValue = KeyString->getValue(KeyStorage);
|
|
|
365 llvm::yaml::Node *Value = NextKeyValue.getValue();
|
|
|
366 if (!Value) {
|
|
|
367 ErrorMessage = "Expected value.";
|
|
|
368 return false;
|
|
|
369 }
|
|
|
370 auto *ValueString = dyn_cast<llvm::yaml::ScalarNode>(Value);
|
|
|
371 auto *SequenceString = dyn_cast<llvm::yaml::SequenceNode>(Value);
|
|
|
372 if (KeyValue == "arguments" && !SequenceString) {
|
|
|
373 ErrorMessage = "Expected sequence as value.";
|
|
|
374 return false;
|
|
|
375 } else if (KeyValue != "arguments" && !ValueString) {
|
|
|
376 ErrorMessage = "Expected string as value.";
|
|
|
377 return false;
|
|
|
378 }
|
|
|
379 if (KeyValue == "directory") {
|
|
|
380 Directory = ValueString;
|
|
|
381 } else if (KeyValue == "arguments") {
|
|
|
382 Command = std::vector<llvm::yaml::ScalarNode *>();
|
|
|
383 for (auto &Argument : *SequenceString) {
|
|
|
384 auto *Scalar = dyn_cast<llvm::yaml::ScalarNode>(&Argument);
|
|
|
385 if (!Scalar) {
|
|
|
386 ErrorMessage = "Only strings are allowed in 'arguments'.";
|
|
|
387 return false;
|
|
|
388 }
|
|
|
389 Command->push_back(Scalar);
|
|
|
390 }
|
|
|
391 } else if (KeyValue == "command") {
|
|
|
392 if (!Command)
|
|
|
393 Command = std::vector<llvm::yaml::ScalarNode *>(1, ValueString);
|
|
|
394 } else if (KeyValue == "file") {
|
|
|
395 File = ValueString;
|
|
|
396 } else if (KeyValue == "output") {
|
|
|
397 Output = ValueString;
|
|
|
398 } else {
|
|
|
399 ErrorMessage = ("Unknown key: \"" +
|
|
|
400 KeyString->getRawValue() + "\"").str();
|
|
|
401 return false;
|
|
|
402 }
|
|
|
403 }
|
|
|
404 if (!File) {
|
|
|
405 ErrorMessage = "Missing key: \"file\".";
|
|
|
406 return false;
|
|
|
407 }
|
|
|
408 if (!Command) {
|
|
|
409 ErrorMessage = "Missing key: \"command\" or \"arguments\".";
|
|
|
410 return false;
|
|
|
411 }
|
|
|
412 if (!Directory) {
|
|
|
413 ErrorMessage = "Missing key: \"directory\".";
|
|
|
414 return false;
|
|
|
415 }
|
|
|
416 SmallString<8> FileStorage;
|
|
|
417 StringRef FileName = File->getValue(FileStorage);
|
|
|
418 SmallString<128> NativeFilePath;
|
|
|
419 if (llvm::sys::path::is_relative(FileName)) {
|
|
|
420 SmallString<8> DirectoryStorage;
|
|
|
421 SmallString<128> AbsolutePath(
|
|
|
422 Directory->getValue(DirectoryStorage));
|
|
|
423 llvm::sys::path::append(AbsolutePath, FileName);
|
|
|
424 llvm::sys::path::remove_dots(AbsolutePath, /*remove_dot_dot=*/ true);
|
|
|
425 llvm::sys::path::native(AbsolutePath, NativeFilePath);
|
|
|
426 } else {
|
|
|
427 llvm::sys::path::native(FileName, NativeFilePath);
|
|
|
428 }
|
|
|
429 auto Cmd = CompileCommandRef(Directory, File, *Command, Output);
|
|
|
430 IndexByFile[NativeFilePath].push_back(Cmd);
|
|
|
431 AllCommands.push_back(Cmd);
|
|
|
432 MatchTrie.insert(NativeFilePath);
|
|
|
433 }
|
|
|
434 return true;
|
|
|
435 }
|
