CbC/CbC_llvm: clang-tools-extra/clangd/GlobalCompilationDatabase.cpp comparison

comparison clang-tools-extra/clangd/GlobalCompilationDatabase.cpp @ 221:79ff65ed7e25

LLVM12 Original

author	Shinji KONO <kono@ie.u-ryukyu.ac.jp>
date	Tue, 15 Jun 2021 19:15:29 +0900
parents	0572611fdcc8
children	c4bab56944e8

comparison

equal deleted inserted replaced

-:42394fc6a535
+:79ff65ed7e25
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "GlobalCompilationDatabase.h"
+#include "Config.h"
 #include "FS.h"
+#include "SourceCode.h"
 #include "support/Logger.h"
 #include "support/Path.h"
+#include "support/Threading.h"
+#include "support/ThreadsafeFS.h"
 #include "clang/Frontend/CompilerInvocation.h"
 #include "clang/Tooling/ArgumentsAdjusters.h"
 #include "clang/Tooling/CompilationDatabase.h"
+#include "clang/Tooling/CompilationDatabasePluginRegistry.h"
+#include "clang/Tooling/JSONCompilationDatabase.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/Optional.h"
+#include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringMap.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
+#include "llvm/Support/VirtualFileSystem.h"
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <mutex>
 #include <string>
 #include <tuple>
 #include <vector>
 namespace clang {
 // Runs the given action on all parent directories of filename, starting from
 // deepest directory and going up to root. Stops whenever action succeeds.
 void actOnAllParentDirectories(PathRef FileName,
 llvm::function_ref<bool(PathRef)> Action) {
-for (auto Path = llvm::sys::path::parent_path(FileName);
+for (auto Path = absoluteParent(FileName); !Path.empty() && !Action(Path);
-!Path.empty() && !Action(Path);
+Path = absoluteParent(Path))
-Path = llvm::sys::path::parent_path(Path))
 ;
 }
 } // namespace
 /*Output=*/"");
 Cmd.Heuristic = "clangd fallback";
 return Cmd;
 }
+// Loads and caches the CDB from a single directory.
+//
+// This class is threadsafe, which is to say we have independent locks for each
+// directory we're searching for a CDB.
+// Loading is deferred until first access.
+//
+// The DirectoryBasedCDB keeps a map from path => DirectoryCache.
+// Typical usage is to:
+//  - 1) determine all the paths that might be searched
+//  - 2) acquire the map lock and get-or-create all the DirectoryCache entries
+//  - 3) release the map lock and query the caches as desired
+class DirectoryBasedGlobalCompilationDatabase::DirectoryCache {
+using stopwatch = std::chrono::steady_clock;
+// CachedFile is used to read a CDB file on disk (e.g. compile_commands.json).
+// It specializes in being able to quickly bail out if the file is unchanged,
+// which is the common case.
+// Internally, it stores file metadata so a stat() can verify it's unchanged.
+// We don't actually cache the content as it's not needed - if the file is
+// unchanged then the previous CDB is valid.
+struct CachedFile {
+CachedFile(llvm::StringRef Parent, llvm::StringRef Rel) {
+llvm::SmallString<256> Path = Parent;
+llvm::sys::path::append(Path, Rel);
+this->Path = Path.str().str();
+}
+std::string Path;
+size_t Size = NoFileCached;
+llvm::sys::TimePoint<> ModifiedTime;
+FileDigest ContentHash;
+static constexpr size_t NoFileCached = -1;
+struct LoadResult {
+enum {
+FileNotFound,
+TransientError,
+FoundSameData,
+FoundNewData,
+} Result;
+std::unique_ptr<llvm::MemoryBuffer> Buffer; // Set only if FoundNewData
+};
+LoadResult load(llvm::vfs::FileSystem &FS, bool HasOldData);
+};
+// If we've looked for a CDB here and found none, the time when that happened.
+// (Atomics make it possible for get() to return without taking a lock)
+std::atomic<stopwatch::rep> NoCDBAt = {
+stopwatch::time_point::min().time_since_epoch().count()};
+// Guards the following cache state.
+std::mutex Mu;
+// When was the cache last known to be in sync with disk state?
+stopwatch::time_point CachePopulatedAt = stopwatch::time_point::min();
+// Whether a new CDB has been loaded but not broadcast yet.
+bool NeedsBroadcast = false;
+// Last loaded CDB, meaningful if CachePopulatedAt was ever set.
+// shared_ptr so we can overwrite this when callers are still using the CDB.
+std::shared_ptr<tooling::CompilationDatabase> CDB;
+// File metadata for the CDB files we support tracking directly.
+CachedFile CompileCommandsJson;
+CachedFile BuildCompileCommandsJson;
+CachedFile CompileFlagsTxt;
+// CachedFile member corresponding to CDB.
+//   CDB  | ACF  | Scenario
+//   null | null | no CDB found, or initial empty cache
+//   set  | null | CDB was loaded via generic plugin interface
+//   null | set  | found known CDB file, but parsing it failed
+//   set  | set  | CDB was parsed from a known file
+CachedFile *ActiveCachedFile = nullptr;
+public:
+DirectoryCache(llvm::StringRef Path)
+: CompileCommandsJson(Path, "compile_commands.json"),
+BuildCompileCommandsJson(Path, "build/compile_commands.json"),
+CompileFlagsTxt(Path, "compile_flags.txt"), Path(Path) {
+assert(llvm::sys::path::is_absolute(Path));
+}
+// Absolute canonical path that we're the cache for. (Not case-folded).
+const std::string Path;
+// Get the CDB associated with this directory.
+// ShouldBroadcast:
+//  - as input, signals whether the caller is willing to broadcast a
+//    newly-discovered CDB. (e.g. to trigger background indexing)
+//  - as output, signals whether the caller should do so.
+// (If a new CDB is discovered and ShouldBroadcast is false, we mark the
+// CDB as needing broadcast, and broadcast it next time we can).
+std::shared_ptr<const tooling::CompilationDatabase>
+get(const ThreadsafeFS &TFS, bool &ShouldBroadcast,
+stopwatch::time_point FreshTime, stopwatch::time_point FreshTimeMissing) {
+// Fast path for common case without taking lock.
+if (stopwatch::time_point(stopwatch::duration(NoCDBAt.load())) >
+FreshTimeMissing) {
+ShouldBroadcast = false;
+return nullptr;
+}
+std::lock_guard<std::mutex> Lock(Mu);
+auto RequestBroadcast = llvm::make_scope_exit([&, OldCDB(CDB.get())] {
+// If we loaded a new CDB, it should be broadcast at some point.
+if (CDB != nullptr && CDB.get() != OldCDB)
+NeedsBroadcast = true;
+else if (CDB == nullptr) // nothing to broadcast anymore!
+NeedsBroadcast = false;
+// If we have something to broadcast, then do so iff allowed.
+if (!ShouldBroadcast)
+return;
+ShouldBroadcast = NeedsBroadcast;
+NeedsBroadcast = false;
+});
+// If our cache is valid, serve from it.
+if (CachePopulatedAt > FreshTime)
+return CDB;
+if (/*MayCache=*/load(*TFS.view(/*CWD=*/llvm::None))) {
+// Use new timestamp, as loading may be slow.
+CachePopulatedAt = stopwatch::now();
+NoCDBAt.store((CDB ? stopwatch::time_point::min() : CachePopulatedAt)
+.time_since_epoch()
+.count());
+}
+return CDB;
+}
+private:
+// Updates `CDB` from disk state. Returns false on failure.
+bool load(llvm::vfs::FileSystem &FS);
+};
+DirectoryBasedGlobalCompilationDatabase::DirectoryCache::CachedFile::LoadResult
+DirectoryBasedGlobalCompilationDatabase::DirectoryCache::CachedFile::load(
+llvm::vfs::FileSystem &FS, bool HasOldData) {
+auto Stat = FS.status(Path);
+if (!Stat || !Stat->isRegularFile()) {
+Size = NoFileCached;
+ContentHash = {};
+return {LoadResult::FileNotFound, nullptr};
+}
+// If both the size and mtime match, presume unchanged without reading.
+if (HasOldData && Stat->getLastModificationTime() == ModifiedTime &&
+Stat->getSize() == Size)
+return {LoadResult::FoundSameData, nullptr};
+auto Buf = FS.getBufferForFile(Path);
+if (!Buf || (*Buf)->getBufferSize() != Stat->getSize()) {
+// Don't clear the cache - possible we're seeing inconsistent size as the
+// file is being recreated. If it ends up identical later, great!
+//
+// This isn't a complete solution: if we see a partial file but stat/read
+// agree on its size, we're ultimately going to have spurious CDB reloads.
+// May be worth fixing if generators don't write atomically (CMake does).
+elog("Failed to read {0}: {1}", Path,
+Buf ? "size changed" : Buf.getError().message());
+return {LoadResult::TransientError, nullptr};
+}
+FileDigest NewContentHash = digest((*Buf)->getBuffer());
+if (HasOldData && NewContentHash == ContentHash) {
+// mtime changed but data is the same: avoid rebuilding the CDB.
+ModifiedTime = Stat->getLastModificationTime();
+return {LoadResult::FoundSameData, nullptr};
+}
+Size = (*Buf)->getBufferSize();
+ModifiedTime = Stat->getLastModificationTime();
+ContentHash = NewContentHash;
+return {LoadResult::FoundNewData, std::move(*Buf)};
+}
+// Adapt CDB-loading functions to a common interface for DirectoryCache::load().
+static std::unique_ptr<tooling::CompilationDatabase>
+parseJSON(PathRef Path, llvm::StringRef Data, std::string &Error) {
+if (auto CDB = tooling::JSONCompilationDatabase::loadFromBuffer(
+Data, Error, tooling::JSONCommandLineSyntax::AutoDetect)) {
+// FS used for expanding response files.
+// FIXME: ExpandResponseFilesDatabase appears not to provide the usual
+// thread-safety guarantees, as the access to FS is not locked!
+// For now, use the real FS, which is known to be threadsafe (if we don't
+// use/change working directory, which ExpandResponseFilesDatabase doesn't).
+auto FS = llvm::vfs::getRealFileSystem();
+return tooling::inferTargetAndDriverMode(
+tooling::inferMissingCompileCommands(
+expandResponseFiles(std::move(CDB), std::move(FS))));
+}
+return nullptr;
+}
+static std::unique_ptr<tooling::CompilationDatabase>
+parseFixed(PathRef Path, llvm::StringRef Data, std::string &Error) {
+return tooling::FixedCompilationDatabase::loadFromBuffer(
+llvm::sys::path::parent_path(Path), Data, Error);
+}
+bool DirectoryBasedGlobalCompilationDatabase::DirectoryCache::load(
+llvm::vfs::FileSystem &FS) {
+dlog("Probing directory {0}", Path);
+std::string Error;
+// Load from the specially-supported compilation databases (JSON + Fixed).
+// For these, we know the files they read and cache their metadata so we can
+// cheaply validate whether they've changed, and hot-reload if they have.
+// (As a bonus, these are also VFS-clean)!
+struct CDBFile {
+CachedFile *File;
+// Wrapper for {Fixed,JSON}CompilationDatabase::loadFromBuffer.
+llvm::function_ref<std::unique_ptr<tooling::CompilationDatabase>(
+PathRef,
+/*Data*/ llvm::StringRef,
+/*ErrorMsg*/ std::string &)>
+Parser;
+};
+for (const auto &Entry : {CDBFile{&CompileCommandsJson, parseJSON},
+CDBFile{&BuildCompileCommandsJson, parseJSON},
+CDBFile{&CompileFlagsTxt, parseFixed}}) {
+bool Active = ActiveCachedFile == Entry.File;
+auto Loaded = Entry.File->load(FS, Active);
+switch (Loaded.Result) {
+case CachedFile::LoadResult::FileNotFound:
+if (Active) {
+log("Unloaded compilation database from {0}", Entry.File->Path);
+ActiveCachedFile = nullptr;
+CDB = nullptr;
+}
+// Continue looking at other candidates.
+break;
+case CachedFile::LoadResult::TransientError:
+// File existed but we couldn't read it. Reuse the cache, retry later.
+return false; // Load again next time.
+case CachedFile::LoadResult::FoundSameData:
+assert(Active && "CachedFile may not return 'same data' if !HasOldData");
+// This is the critical file, and it hasn't changed.
+return true;
+case CachedFile::LoadResult::FoundNewData:
+// We have a new CDB!
+CDB = Entry.Parser(Entry.File->Path, Loaded.Buffer->getBuffer(), Error);
+if (CDB)
+log("{0} compilation database from {1}", Active ? "Reloaded" : "Loaded",
+Entry.File->Path);
+else
+elog("Failed to load compilation database from {0}: {1}",
+Entry.File->Path, Error);
+ActiveCachedFile = Entry.File;
+return true;
+}
+}
+// Fall back to generic handling of compilation databases.
+// We don't know what files they read, so can't efficiently check whether
+// they need to be reloaded. So we never do that.
+// FIXME: the interface doesn't provide a way to virtualize FS access.
+// Don't try these more than once. If we've scanned before, we're done.
+if (CachePopulatedAt > stopwatch::time_point::min())
+return true;
+for (const auto &Entry :
+tooling::CompilationDatabasePluginRegistry::entries()) {
+// Avoid duplicating the special cases handled above.
+if (Entry.getName() == "fixed-compilation-database" ||
+Entry.getName() == "json-compilation-database")
+continue;
+auto Plugin = Entry.instantiate();
+if (auto CDB = Plugin->loadFromDirectory(Path, Error)) {
+log("Loaded compilation database from {0} with plugin {1}", Path,
+Entry.getName());
+this->CDB = std::move(CDB);
+return true;
+}
+// Don't log Error here, it's usually just "couldn't find <file>".
+}
+dlog("No compilation database at {0}", Path);
+return true;
+}
 DirectoryBasedGlobalCompilationDatabase::
-DirectoryBasedGlobalCompilationDatabase(
+DirectoryBasedGlobalCompilationDatabase(const Options &Opts)
-llvm::Optional<Path> CompileCommandsDir)
+: Opts(Opts), Broadcaster(std::make_unique<BroadcastThread>(*this)) {
-: CompileCommandsDir(std::move(CompileCommandsDir)) {}
+if (!this->Opts.ContextProvider)
+this->Opts.ContextProvider = [](llvm::StringRef) {
+return Context::current().clone();
+};
+}
 DirectoryBasedGlobalCompilationDatabase::
 ~DirectoryBasedGlobalCompilationDatabase() = default;
 llvm::Optional<tooling::CompileCommand>
 DirectoryBasedGlobalCompilationDatabase::getCompileCommand(PathRef File) const {
 CDBLookupRequest Req;
 Req.FileName = File;
 Req.ShouldBroadcast = true;
+auto Now = std::chrono::steady_clock::now();
+Req.FreshTime = Now - Opts.RevalidateAfter;
+Req.FreshTimeMissing = Now - Opts.RevalidateMissingAfter;
 auto Res = lookupCDB(Req);
 if (!Res) {
 log("Failed to find compilation database for {0}", File);
 return llvm::None;
 return std::move(Candidates.front());
 return None;
 }
-// For platforms where paths are case-insensitive (but case-preserving),
+std::vector<DirectoryBasedGlobalCompilationDatabase::DirectoryCache *>
-// we need to do case-insensitive comparisons and use lowercase keys.
+DirectoryBasedGlobalCompilationDatabase::getDirectoryCaches(
-// FIXME: Make Path a real class with desired semantics instead.
+llvm::ArrayRef<llvm::StringRef> Dirs) const {
-//        This class is not the only place this problem exists.
+std::vector<std::string> FoldedDirs;
-// FIXME: Mac filesystems default to case-insensitive, but may be sensitive.
+FoldedDirs.reserve(Dirs.size());
+for (const auto &Dir : Dirs) {
-static std::string maybeCaseFoldPath(PathRef Path) {
+#ifndef NDEBUG
-#if defined(_WIN32) || defined(__APPLE__)
+if (!llvm::sys::path::is_absolute(Dir))
-return Path.lower();
+elog("Trying to cache CDB for relative {0}");
-#else
-return std::string(Path);
 #endif
-}
+FoldedDirs.push_back(maybeCaseFoldPath(Dir));
+}
-static bool pathEqual(PathRef A, PathRef B) {
-#if defined(_WIN32) || defined(__APPLE__)
+std::vector<DirectoryCache *> Ret;
-return A.equals_lower(B);
+Ret.reserve(Dirs.size());
-#else
-return A == B;
+std::lock_guard<std::mutex> Lock(DirCachesMutex);
-#endif
+for (unsigned I = 0; I < Dirs.size(); ++I)
-}
+Ret.push_back(&DirCaches.try_emplace(FoldedDirs[I], Dirs[I]).first->second);
+return Ret;
-DirectoryBasedGlobalCompilationDatabase::CachedCDB &
-DirectoryBasedGlobalCompilationDatabase::getCDBInDirLocked(PathRef Dir) const {
-// FIXME(ibiryukov): Invalidate cached compilation databases on changes
-auto Key = maybeCaseFoldPath(Dir);
-auto R = CompilationDatabases.try_emplace(Key);
-if (R.second) { // Cache miss, try to load CDB.
-CachedCDB &Entry = R.first->second;
-std::string Error;
-Entry.Path = std::string(Dir);
-Entry.CDB = tooling::CompilationDatabase::loadFromDirectory(Dir, Error);
-// Check for $src/build, the conventional CMake build root.
-// Probe existence first to avoid each plugin doing IO if it doesn't exist.
-if (!CompileCommandsDir && !Entry.CDB) {
-llvm::SmallString<256> BuildDir = Dir;
-llvm::sys::path::append(BuildDir, "build");
-if (llvm::sys::fs::is_directory(BuildDir)) {
-vlog("Found candidate build directory {0}", BuildDir);
-Entry.CDB =
-tooling::CompilationDatabase::loadFromDirectory(BuildDir, Error);
-}
-}
-if (Entry.CDB)
-log("Loaded compilation database from {0}", Dir);
-}
-return R.first->second;
 }
 llvm::Optional<DirectoryBasedGlobalCompilationDatabase::CDBLookupResult>
 DirectoryBasedGlobalCompilationDatabase::lookupCDB(
 CDBLookupRequest Request) const {
 assert(llvm::sys::path::is_absolute(Request.FileName) &&
 "path must be absolute");
+std::string Storage;
+std::vector<llvm::StringRef> SearchDirs;
+if (Opts.CompileCommandsDir) // FIXME: unify this case with config.
+SearchDirs = {Opts.CompileCommandsDir.getValue()};
+else {
+WithContext WithProvidedContext(Opts.ContextProvider(Request.FileName));
+const auto &Spec = Config::current().CompileFlags.CDBSearch;
+switch (Spec.Policy) {
+case Config::CDBSearchSpec::NoCDBSearch:
+return llvm::None;
+case Config::CDBSearchSpec::FixedDir:
+Storage = Spec.FixedCDBPath.getValue();
+SearchDirs = {Storage};
+break;
+case Config::CDBSearchSpec::Ancestors:
+// Traverse the canonical version to prevent false positives. i.e.:
+// src/build/../a.cc can detect a CDB in /src/build if not
+// canonicalized.
+Storage = removeDots(Request.FileName);
+actOnAllParentDirectories(Storage, [&](llvm::StringRef Dir) {
+SearchDirs.push_back(Dir);
+return false;
+});
+}
+}
+std::shared_ptr<const tooling::CompilationDatabase> CDB = nullptr;
 bool ShouldBroadcast = false;
+DirectoryCache *DirCache = nullptr;
+for (DirectoryCache *Candidate : getDirectoryCaches(SearchDirs)) {
+bool CandidateShouldBroadcast = Request.ShouldBroadcast;
+if ((CDB = Candidate->get(Opts.TFS, CandidateShouldBroadcast,
+Request.FreshTime, Request.FreshTimeMissing))) {
+DirCache = Candidate;
+ShouldBroadcast = CandidateShouldBroadcast;
+break;
+}
+}
+if (!CDB)
+return llvm::None;
 CDBLookupResult Result;
+Result.CDB = std::move(CDB);
-{
+Result.PI.SourceRoot = DirCache->Path;
-std::lock_guard<std::mutex> Lock(Mutex);
-CachedCDB *Entry = nullptr;
-if (CompileCommandsDir) {
-Entry = &getCDBInDirLocked(*CompileCommandsDir);
-} else {
-// Traverse the canonical version to prevent false positives. i.e.:
-// src/build/../a.cc can detect a CDB in /src/build if not canonicalized.
-// FIXME(sammccall): this loop is hot, use a union-find-like structure.
-actOnAllParentDirectories(removeDots(Request.FileName),
-[&](PathRef Path) {
-Entry = &getCDBInDirLocked(Path);
-return Entry->CDB != nullptr;
-});
-}
-if (!Entry || !Entry->CDB)
-return llvm::None;
-// Mark CDB as broadcasted to make sure discovery is performed once.
-if (Request.ShouldBroadcast && !Entry->SentBroadcast) {
-Entry->SentBroadcast = true;
-ShouldBroadcast = true;
-}
-Result.CDB = Entry->CDB.get();
-Result.PI.SourceRoot = Entry->Path;
-}
-// FIXME: Maybe make the following part async, since this can block retrieval
-// of compile commands.
 if (ShouldBroadcast)
 broadcastCDB(Result);
 return Result;
 }
+// The broadcast thread announces files with new compile commands to the world.
+// Primarily this is used to enqueue them for background indexing.
+//
+// It's on a separate thread because:
+//  - otherwise it would block the first parse of the initial file
+//  - we need to enumerate all files in the CDB, of which there are many
+//  - we (will) have to evaluate config for every file in the CDB, which is slow
+class DirectoryBasedGlobalCompilationDatabase::BroadcastThread {
+class Filter;
+DirectoryBasedGlobalCompilationDatabase &Parent;
+std::mutex Mu;
+std::condition_variable CV;
+// Shutdown flag (CV is notified after writing).
+// This is atomic so that broadcasts can also observe it and abort early.
+std::atomic<bool> ShouldStop = {false};
+struct Task {
+CDBLookupResult Lookup;
+Context Ctx;
+};
+std::deque<Task> Queue;
+llvm::Optional<Task> ActiveTask;
+std::thread Thread; // Must be last member.
+// Thread body: this is just the basic queue procesing boilerplate.
+void run() {
+std::unique_lock<std::mutex> Lock(Mu);
+while (true) {
+bool Stopping = false;
+CV.wait(Lock, [&] {
+return (Stopping = ShouldStop.load(std::memory_order_acquire)) ||
+!Queue.empty();
+});
+if (Stopping) {
+Queue.clear();
+CV.notify_all();
+return;
+}
+ActiveTask = std::move(Queue.front());
+Queue.pop_front();
+Lock.unlock();
+{
+WithContext WithCtx(std::move(ActiveTask->Ctx));
+process(ActiveTask->Lookup);
+}
+Lock.lock();
+ActiveTask.reset();
+CV.notify_all();
+}
+}
+// Inspects a new CDB and broadcasts the files it owns.
+void process(const CDBLookupResult &T);
+public:
+BroadcastThread(DirectoryBasedGlobalCompilationDatabase &Parent)
+: Parent(Parent), Thread([this] { run(); }) {}
+void enqueue(CDBLookupResult Lookup) {
+{
+assert(!Lookup.PI.SourceRoot.empty());
+std::lock_guard<std::mutex> Lock(Mu);
+// New CDB takes precedence over any queued one for the same directory.
+llvm::erase_if(Queue, [&](const Task &T) {
+return T.Lookup.PI.SourceRoot == Lookup.PI.SourceRoot;
+});
+Queue.push_back({std::move(Lookup), Context::current().clone()});
+}
+CV.notify_all();
+}
+bool blockUntilIdle(Deadline Timeout) {
+std::unique_lock<std::mutex> Lock(Mu);
+return wait(Lock, CV, Timeout,
+[&] { return Queue.empty() && !ActiveTask.hasValue(); });
+}
+~BroadcastThread() {
+{
+std::lock_guard<std::mutex> Lock(Mu);
+ShouldStop.store(true, std::memory_order_release);
+}
+CV.notify_all();
+Thread.join();
+}
+};
+// The DirBasedCDB associates each file with a specific CDB.
+// When a CDB is discovered, it may claim to describe files that we associate
+// with a different CDB. We do not want to broadcast discovery of these, and
+// trigger background indexing of them.
+//
+// We must filter the list, and check whether they are associated with this CDB.
+// This class attempts to do so efficiently.
+//
+// Roughly, it:
+//  - loads the config for each file, and determines the relevant search path
+//  - gathers all directories that are part of any search path
+//  - (lazily) checks for a CDB in each such directory at most once
+//  - walks the search path for each file and determines whether to include it.
+class DirectoryBasedGlobalCompilationDatabase::BroadcastThread::Filter {
+llvm::StringRef ThisDir;
+DirectoryBasedGlobalCompilationDatabase &Parent;
+// Keep track of all directories we might check for CDBs.
+struct DirInfo {
+DirectoryCache *Cache = nullptr;
+enum { Unknown, Missing, TargetCDB, OtherCDB } State = Unknown;
+DirInfo *Parent = nullptr;
+};
+llvm::StringMap<DirInfo> Dirs;
+// A search path starts at a directory, and either includes ancestors or not.
+using SearchPath = llvm::PointerIntPair<DirInfo *, 1>;
+// Add all ancestor directories of FilePath to the tracked set.
+// Returns the immediate parent of the file.
+DirInfo *addParents(llvm::StringRef FilePath) {
+DirInfo *Leaf = nullptr;
+DirInfo *Child = nullptr;
+actOnAllParentDirectories(FilePath, [&](llvm::StringRef Dir) {
+auto &Info = Dirs[Dir];
+// If this is the first iteration, then this node is the overall result.
+if (!Leaf)
+Leaf = &Info;
+// Fill in the parent link from the previous iteration to this parent.
+if (Child)
+Child->Parent = &Info;
+// Keep walking, whether we inserted or not, if parent link is missing.
+// (If it's present, parent links must be present up to the root, so stop)
+Child = &Info;
+return Info.Parent != nullptr;
+});
+return Leaf;
+}
+// Populates DirInfo::Cache (and State, if it is TargetCDB).
+void grabCaches() {
+// Fast path out if there were no files, or CDB loading is off.
+if (Dirs.empty())
+return;
+std::vector<llvm::StringRef> DirKeys;
+std::vector<DirInfo *> DirValues;
+DirKeys.reserve(Dirs.size() + 1);
+DirValues.reserve(Dirs.size());
+for (auto &E : Dirs) {
+DirKeys.push_back(E.first());
+DirValues.push_back(&E.second);
+}
+// Also look up the cache entry for the CDB we're broadcasting.
+// Comparing DirectoryCache pointers is more robust than checking string
+// equality, e.g. reuses the case-sensitivity handling.
+DirKeys.push_back(ThisDir);
+auto DirCaches = Parent.getDirectoryCaches(DirKeys);
+const DirectoryCache *ThisCache = DirCaches.back();
+DirCaches.pop_back();
+DirKeys.pop_back();
+for (unsigned I = 0; I < DirKeys.size(); ++I) {
+DirValues[I]->Cache = DirCaches[I];
+if (DirCaches[I] == ThisCache)
+DirValues[I]->State = DirInfo::TargetCDB;
+}
+}
+// Should we include a file from this search path?
+bool shouldInclude(SearchPath P) {
+DirInfo *Info = P.getPointer();
+if (!Info)
+return false;
+if (Info->State == DirInfo::Unknown) {
+assert(Info->Cache && "grabCaches() should have filled this");
+// Given that we know that CDBs have been moved/generated, don't trust
+// caches. (This should be rare, so it's OK to add a little latency).
+constexpr auto IgnoreCache = std::chrono::steady_clock::time_point::max();
+// Don't broadcast CDBs discovered while broadcasting!
+bool ShouldBroadcast = false;
+bool Exists =
+nullptr != Info->Cache->get(Parent.Opts.TFS, ShouldBroadcast,
+/*FreshTime=*/IgnoreCache,
+/*FreshTimeMissing=*/IgnoreCache);
+Info->State = Exists ? DirInfo::OtherCDB : DirInfo::Missing;
+}
+// If we have a CDB, include the file if it's the target CDB only.
+if (Info->State != DirInfo::Missing)
+return Info->State == DirInfo::TargetCDB;
+// If we have no CDB and no relevant parent, don't include the file.
+if (!P.getInt() || !Info->Parent)
+return false;
+// Walk up to the next parent.
+return shouldInclude(SearchPath(Info->Parent, 1));
+}
+public:
+Filter(llvm::StringRef ThisDir,
+DirectoryBasedGlobalCompilationDatabase &Parent)
+: ThisDir(ThisDir), Parent(Parent) {}
+std::vector<std::string> filter(std::vector<std::string> AllFiles,
+std::atomic<bool> &ShouldStop) {
+std::vector<std::string> Filtered;
+// Allow for clean early-exit of the slow parts.
+auto ExitEarly = [&] {
+if (ShouldStop.load(std::memory_order_acquire)) {
+log("Giving up on broadcasting CDB, as we're shutting down");
+Filtered.clear();
+return true;
+}
+return false;
+};
+// Compute search path for each file.
+std::vector<SearchPath> SearchPaths(AllFiles.size());
+for (unsigned I = 0; I < AllFiles.size(); ++I) {
+if (Parent.Opts.CompileCommandsDir) { // FIXME: unify with config
+SearchPaths[I].setPointer(
+&Dirs[Parent.Opts.CompileCommandsDir.getValue()]);
+continue;
+}
+if (ExitEarly()) // loading config may be slow
+return Filtered;
+WithContext WithProvidedContent(Parent.Opts.ContextProvider(AllFiles[I]));
+const Config::CDBSearchSpec &Spec =
+Config::current().CompileFlags.CDBSearch;
+switch (Spec.Policy) {
+case Config::CDBSearchSpec::NoCDBSearch:
+break;
+case Config::CDBSearchSpec::Ancestors:
+SearchPaths[I].setInt(/*Recursive=*/1);
+SearchPaths[I].setPointer(addParents(AllFiles[I]));
+break;
+case Config::CDBSearchSpec::FixedDir:
+SearchPaths[I].setPointer(&Dirs[Spec.FixedCDBPath.getValue()]);
+break;
+}
+}
+// Get the CDB cache for each dir on the search path, but don't load yet.
+grabCaches();
+// Now work out which files we want to keep, loading CDBs where needed.
+for (unsigned I = 0; I < AllFiles.size(); ++I) {
+if (ExitEarly()) // loading CDBs may be slow
+return Filtered;
+if (shouldInclude(SearchPaths[I]))
+Filtered.push_back(std::move(AllFiles[I]));
+}
+return Filtered;
+}
+};
+void DirectoryBasedGlobalCompilationDatabase::BroadcastThread::process(
+const CDBLookupResult &T) {
+vlog("Broadcasting compilation database from {0}", T.PI.SourceRoot);
+std::vector<std::string> GovernedFiles =
+Filter(T.PI.SourceRoot, Parent).filter(T.CDB->getAllFiles(), ShouldStop);
+if (!GovernedFiles.empty())
+Parent.OnCommandChanged.broadcast(std::move(GovernedFiles));
+}
 void DirectoryBasedGlobalCompilationDatabase::broadcastCDB(
 CDBLookupResult Result) const {
 assert(Result.CDB && "Trying to broadcast an invalid CDB!");
+Broadcaster->enqueue(Result);
-std::vector<std::string> AllFiles = Result.CDB->getAllFiles();
+}
-// We assume CDB in CompileCommandsDir owns all of its entries, since we don't
-// perform any search in parent paths whenever it is set.
+bool DirectoryBasedGlobalCompilationDatabase::blockUntilIdle(
-if (CompileCommandsDir) {
+Deadline Timeout) const {
-assert(*CompileCommandsDir == Result.PI.SourceRoot &&
+return Broadcaster->blockUntilIdle(Timeout);
-"Trying to broadcast a CDB outside of CompileCommandsDir!");
-OnCommandChanged.broadcast(std::move(AllFiles));
-return;
-}
-llvm::StringMap<bool> DirectoryHasCDB;
-{
-std::lock_guard<std::mutex> Lock(Mutex);
-// Uniquify all parent directories of all files.
-for (llvm::StringRef File : AllFiles) {
-actOnAllParentDirectories(File, [&](PathRef Path) {
-auto It = DirectoryHasCDB.try_emplace(Path);
-// Already seen this path, and all of its parents.
-if (!It.second)
-return true;
-CachedCDB &Entry = getCDBInDirLocked(Path);
-It.first->second = Entry.CDB != nullptr;
-return pathEqual(Path, Result.PI.SourceRoot);
-});
-}
-}
-std::vector<std::string> GovernedFiles;
-for (llvm::StringRef File : AllFiles) {
-// A file is governed by this CDB if lookup for the file would find it.
-// Independent of whether it has an entry for that file or not.
-actOnAllParentDirectories(File, [&](PathRef Path) {
-if (DirectoryHasCDB.lookup(Path)) {
-if (pathEqual(Path, Result.PI.SourceRoot))
-// Make sure listeners always get a canonical path for the file.
-GovernedFiles.push_back(removeDots(File));
-// Stop as soon as we hit a CDB.
-return true;
-}
-return false;
-});
-}
-OnCommandChanged.broadcast(std::move(GovernedFiles));
 }
 llvm::Optional<ProjectInfo>
 DirectoryBasedGlobalCompilationDatabase::getProjectInfo(PathRef File) const {
 CDBLookupRequest Req;
 Req.FileName = File;
 Req.ShouldBroadcast = false;
+Req.FreshTime = Req.FreshTimeMissing =
+std::chrono::steady_clock::time_point::min();
 auto Res = lookupCDB(Req);
 if (!Res)
 return llvm::None;
 return Res->PI;
 }
 OverlayCDB::OverlayCDB(const GlobalCompilationDatabase *Base,
 std::vector<std::string> FallbackFlags,
 tooling::ArgumentsAdjuster Adjuster)
-: Base(Base), ArgsAdjuster(std::move(Adjuster)),
+: DelegatingCDB(Base), ArgsAdjuster(std::move(Adjuster)),
-FallbackFlags(std::move(FallbackFlags)) {
+FallbackFlags(std::move(FallbackFlags)) {}
-if (Base)
-BaseChanged = Base->watch([this](const std::vector<std::string> Changes) {
-OnCommandChanged.broadcast(Changes);
-});
-}
 llvm::Optional<tooling::CompileCommand>
 OverlayCDB::getCompileCommand(PathRef File) const {
 llvm::Optional<tooling::CompileCommand> Cmd;
 {
 std::lock_guard<std::mutex> Lock(Mutex);
 auto It = Commands.find(removeDots(File));
 if (It != Commands.end())
 Cmd = It->second;
 }
-if (!Cmd && Base)
+if (!Cmd)
-Cmd = Base->getCompileCommand(File);
+Cmd = DelegatingCDB::getCompileCommand(File);
 if (!Cmd)
 return llvm::None;
 if (ArgsAdjuster)
 Cmd->CommandLine = ArgsAdjuster(Cmd->CommandLine, Cmd->Filename);
 return Cmd;
 }
 tooling::CompileCommand OverlayCDB::getFallbackCommand(PathRef File) const {
-auto Cmd = Base ? Base->getFallbackCommand(File)
+auto Cmd = DelegatingCDB::getFallbackCommand(File);
-: GlobalCompilationDatabase::getFallbackCommand(File);
 std::lock_guard<std::mutex> Lock(Mutex);
 Cmd.CommandLine.insert(Cmd.CommandLine.end(), FallbackFlags.begin(),
 FallbackFlags.end());
 if (ArgsAdjuster)
 Cmd.CommandLine = ArgsAdjuster(Cmd.CommandLine, Cmd.Filename);
 Commands.erase(CanonPath);
 }
 OnCommandChanged.broadcast({CanonPath});
 }
-llvm::Optional<ProjectInfo> OverlayCDB::getProjectInfo(PathRef File) const {
+DelegatingCDB::DelegatingCDB(const GlobalCompilationDatabase *Base)
-{
+: Base(Base) {
-std::lock_guard<std::mutex> Lock(Mutex);
-auto It = Commands.find(removeDots(File));
-if (It != Commands.end())
-return ProjectInfo{};
-}
 if (Base)
-return Base->getProjectInfo(File);
+BaseChanged = Base->watch([this](const std::vector<std::string> Changes) {
+OnCommandChanged.broadcast(Changes);
-return llvm::None;
+});
 }
+DelegatingCDB::DelegatingCDB(std::unique_ptr<GlobalCompilationDatabase> Base)
+: DelegatingCDB(Base.get()) {
+BaseOwner = std::move(Base);
+}
+llvm::Optional<tooling::CompileCommand>
+DelegatingCDB::getCompileCommand(PathRef File) const {
+if (!Base)
+return llvm::None;
+return Base->getCompileCommand(File);
+}
+llvm::Optional<ProjectInfo> DelegatingCDB::getProjectInfo(PathRef File) const {
+if (!Base)
+return llvm::None;
+return Base->getProjectInfo(File);
+}
+tooling::CompileCommand DelegatingCDB::getFallbackCommand(PathRef File) const {
+if (!Base)
+return GlobalCompilationDatabase::getFallbackCommand(File);
+return Base->getFallbackCommand(File);
+}
+bool DelegatingCDB::blockUntilIdle(Deadline D) const {
+if (!Base)
+return true;
+return Base->blockUntilIdle(D);
+}
 } // namespace clangd
 } // namespace clang

Mercurial > hg > CbC > CbC_llvm

comparison clang-tools-extra/clangd/GlobalCompilationDatabase.cpp @ 221:79ff65ed7e25