CbC/CbC_llvm: clang-tools-extra/clangd/SemanticHighlighting.cpp comparison

comparison clang-tools-extra/clangd/SemanticHighlighting.cpp @ 221:79ff65ed7e25

LLVM12 Original

author	Shinji KONO <kono@ie.u-ryukyu.ac.jp>
date	Tue, 15 Jun 2021 19:15:29 +0900
parents	0572611fdcc8
children	5f17cb93ff66

comparison

equal deleted inserted replaced

-:42394fc6a535
+:79ff65ed7e25
 //
 //===----------------------------------------------------------------------===//
 #include "SemanticHighlighting.h"
 #include "FindTarget.h"
+#include "HeuristicResolver.h"
 #include "ParsedAST.h"
 #include "Protocol.h"
 #include "SourceCode.h"
 #include "support/Logger.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
+#include "clang/AST/DeclObjC.h"
+#include "clang/AST/DeclTemplate.h"
 #include "clang/AST/DeclarationName.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/AST/Type.h"
 #include "clang/AST/TypeLoc.h"
 namespace {
 /// Some names are not written in the source code and cannot be highlighted,
 /// e.g. anonymous classes. This function detects those cases.
 bool canHighlightName(DeclarationName Name) {
-if (Name.getNameKind() == DeclarationName::CXXConstructorName ||
+switch (Name.getNameKind()) {
-Name.getNameKind() == DeclarationName::CXXUsingDirective)
+case DeclarationName::Identifier: {
-return true;
+auto *II = Name.getAsIdentifierInfo();
-auto *II = Name.getAsIdentifierInfo();
+return II && !II->getName().empty();
-return II && !II->getName().empty();
+}
-}
+case DeclarationName::CXXConstructorName:
+case DeclarationName::CXXDestructorName:
-llvm::Optional<HighlightingKind> kindForType(const Type *TP);
+return true;
-llvm::Optional<HighlightingKind> kindForDecl(const NamedDecl *D) {
+case DeclarationName::ObjCZeroArgSelector:
+case DeclarationName::ObjCOneArgSelector:
+case DeclarationName::ObjCMultiArgSelector:
+// Multi-arg selectors need special handling, and we handle 0/1 arg
+// selectors there too.
+return false;
+case DeclarationName::CXXConversionFunctionName:
+case DeclarationName::CXXOperatorName:
+case DeclarationName::CXXDeductionGuideName:
+case DeclarationName::CXXLiteralOperatorName:
+case DeclarationName::CXXUsingDirective:
+return false;
+}
+llvm_unreachable("invalid name kind");
+}
+llvm::Optional<HighlightingKind> kindForType(const Type *TP,
+const HeuristicResolver *Resolver);
+llvm::Optional<HighlightingKind>
+kindForDecl(const NamedDecl *D, const HeuristicResolver *Resolver) {
 if (auto *USD = dyn_cast<UsingShadowDecl>(D)) {
 if (auto *Target = USD->getTargetDecl())
 D = Target;
 }
 if (auto *TD = dyn_cast<TemplateDecl>(D)) {
 if (auto *Templated = TD->getTemplatedDecl())
 D = Templated;
 }
 if (auto *TD = dyn_cast<TypedefNameDecl>(D)) {
 // We try to highlight typedefs as their underlying type.
-if (auto K = kindForType(TD->getUnderlyingType().getTypePtrOrNull()))
+if (auto K =
+kindForType(TD->getUnderlyingType().getTypePtrOrNull(), Resolver))
 return K;
 // And fallback to a generic kind if this fails.
 return HighlightingKind::Typedef;
 }
 // We highlight class decls, constructor decls and destructor decls as
 // We don't want to highlight lambdas like classes.
 if (RD->isLambda())
 return llvm::None;
 return HighlightingKind::Class;
 }
-if (isa<ClassTemplateDecl>(D) || isa<RecordDecl>(D) ||
+if (isa<ClassTemplateDecl, RecordDecl, CXXConstructorDecl, ObjCInterfaceDecl,
-isa<CXXConstructorDecl>(D))
+ObjCImplementationDecl>(D))
 return HighlightingKind::Class;
+if (isa<ObjCProtocolDecl>(D))
+return HighlightingKind::Interface;
+if (isa<ObjCCategoryDecl>(D))
+return HighlightingKind::Namespace;
 if (auto *MD = dyn_cast<CXXMethodDecl>(D))
 return MD->isStatic() ? HighlightingKind::StaticMethod
 : HighlightingKind::Method;
-if (isa<FieldDecl>(D))
+if (auto *OMD = dyn_cast<ObjCMethodDecl>(D))
+return OMD->isClassMethod() ? HighlightingKind::StaticMethod
+: HighlightingKind::Method;
+if (isa<FieldDecl, ObjCPropertyDecl>(D))
 return HighlightingKind::Field;
 if (isa<EnumDecl>(D))
 return HighlightingKind::Enum;
 if (isa<EnumConstantDecl>(D))
 return HighlightingKind::EnumConstant;
 if (isa<ParmVarDecl>(D))
 return HighlightingKind::Parameter;
-if (auto *VD = dyn_cast<VarDecl>(D))
+if (auto *VD = dyn_cast<VarDecl>(D)) {
+if (isa<ImplicitParamDecl>(VD)) // e.g. ObjC Self
+return llvm::None;
 return VD->isStaticDataMember()
 ? HighlightingKind::StaticField
 : VD->isLocalVarDecl() ? HighlightingKind::LocalVariable
 : HighlightingKind::Variable;
-if (isa<BindingDecl>(D))
+}
-return HighlightingKind::Variable;
+if (const auto *BD = dyn_cast<BindingDecl>(D))
+return BD->getDeclContext()->isFunctionOrMethod()
+? HighlightingKind::LocalVariable
+: HighlightingKind::Variable;
 if (isa<FunctionDecl>(D))
 return HighlightingKind::Function;
 if (isa<NamespaceDecl>(D) || isa<NamespaceAliasDecl>(D) ||
 isa<UsingDirectiveDecl>(D))
 return HighlightingKind::Namespace;
 if (isa<TemplateTemplateParmDecl>(D) || isa<TemplateTypeParmDecl>(D) ||
 isa<NonTypeTemplateParmDecl>(D))
 return HighlightingKind::TemplateParameter;
 if (isa<ConceptDecl>(D))
 return HighlightingKind::Concept;
+if (const auto *UUVD = dyn_cast<UnresolvedUsingValueDecl>(D)) {
+auto Targets = Resolver->resolveUsingValueDecl(UUVD);
+if (!Targets.empty()) {
+return kindForDecl(Targets[0], Resolver);
+}
+return HighlightingKind::Unknown;
+}
 return llvm::None;
 }
-llvm::Optional<HighlightingKind> kindForType(const Type *TP) {
+llvm::Optional<HighlightingKind>
+kindForType(const Type *TP, const HeuristicResolver *Resolver) {
 if (!TP)
 return llvm::None;
 if (TP->isBuiltinType()) // Builtins are special, they do not have decls.
 return HighlightingKind::Primitive;
 if (auto *TD = dyn_cast<TemplateTypeParmType>(TP))
-return kindForDecl(TD->getDecl());
+return kindForDecl(TD->getDecl(), Resolver);
+if (isa<ObjCObjectPointerType>(TP))
+return HighlightingKind::Class;
 if (auto *TD = TP->getAsTagDecl())
-return kindForDecl(TD);
+return kindForDecl(TD, Resolver);
 return llvm::None;
 }
-llvm::Optional<HighlightingKind> kindForReference(const ReferenceLoc &R) {
+// Whether T is const in a loose sense - is a variable with this type readonly?
-llvm::Optional<HighlightingKind> Result;
+bool isConst(QualType T) {
-for (const NamedDecl *Decl : R.Targets) {
+if (T.isNull() || T->isDependentType())
-if (!canHighlightName(Decl->getDeclName()))
+return false;
-return llvm::None;
+T = T.getNonReferenceType();
-auto Kind = kindForDecl(Decl);
+if (T.isConstQualified())
-if (!Kind || (Result && Kind != Result))
+return true;
-return llvm::None;
+if (const auto *AT = T->getAsArrayTypeUnsafe())
-Result = Kind;
+return isConst(AT->getElementType());
-}
+if (isConst(T->getPointeeType()))
-return Result;
+return true;
+return false;
+}
+// Whether D is const in a loose sense (should it be highlighted as such?)
+// FIXME: This is separate from whether *a particular usage* can mutate D.
+//        We may want V in V.size() to be readonly even if V is mutable.
+bool isConst(const Decl *D) {
+if (llvm::isa<EnumConstantDecl>(D) || llvm::isa<NonTypeTemplateParmDecl>(D))
+return true;
+if (llvm::isa<FieldDecl>(D) || llvm::isa<VarDecl>(D) ||
+llvm::isa<MSPropertyDecl>(D) || llvm::isa<BindingDecl>(D)) {
+if (isConst(llvm::cast<ValueDecl>(D)->getType()))
+return true;
+}
+if (const auto *OCPD = llvm::dyn_cast<ObjCPropertyDecl>(D)) {
+if (OCPD->isReadOnly())
+return true;
+}
+if (const auto *MPD = llvm::dyn_cast<MSPropertyDecl>(D)) {
+if (!MPD->hasSetter())
+return true;
+}
+if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D)) {
+if (CMD->isConst())
+return true;
+}
+return false;
+}
+// "Static" means many things in C++, only some get the "static" modifier.
+//
+// Meanings that do:
+// - Members associated with the class rather than the instance.
+//   This is what 'static' most often means across languages.
+// - static local variables
+//   These are similarly "detached from their context" by the static keyword.
+//   In practice, these are rarely used inside classes, reducing confusion.
+//
+// Meanings that don't:
+// - Namespace-scoped variables, which have static storage class.
+//   This is implicit, so the keyword "static" isn't so strongly associated.
+//   If we want a modifier for these, "global scope" is probably the concept.
+// - Namespace-scoped variables/functions explicitly marked "static".
+//   There the keyword changes *linkage* , which is a totally different concept.
+//   If we want to model this, "file scope" would be a nice modifier.
+//
+// This is confusing, and maybe we should use another name, but because "static"
+// is a standard LSP modifier, having one with that name has advantages.
+bool isStatic(const Decl *D) {
+if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D))
+return CMD->isStatic();
+if (const VarDecl *VD = llvm::dyn_cast<VarDecl>(D))
+return VD->isStaticDataMember() || VD->isStaticLocal();
+if (const auto *OPD = llvm::dyn_cast<ObjCPropertyDecl>(D))
+return OPD->isClassProperty();
+if (const auto *OMD = llvm::dyn_cast<ObjCMethodDecl>(D))
+return OMD->isClassMethod();
+return false;
+}
+bool isAbstract(const Decl *D) {
+if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D))
+return CMD->isPure();
+if (const auto *CRD = llvm::dyn_cast<CXXRecordDecl>(D))
+return CRD->hasDefinition() && CRD->isAbstract();
+return false;
+}
+bool isDependent(const Decl *D) {
+if (isa<UnresolvedUsingValueDecl>(D))
+return true;
+return false;
+}
+/// Returns true if `Decl` is considered to be from a default/system library.
+/// This currently checks the systemness of the file by include type, although
+/// different heuristics may be used in the future (e.g. sysroot paths).
+bool isDefaultLibrary(const Decl *D) {
+SourceLocation Loc = D->getLocation();
+if (!Loc.isValid())
+return false;
+return D->getASTContext().getSourceManager().isInSystemHeader(Loc);
+}
+bool isDefaultLibrary(const Type *T) {
+if (!T)
+return false;
+const Type *Underlying = T->getPointeeOrArrayElementType();
+if (Underlying->isBuiltinType())
+return true;
+if (auto *TD = dyn_cast<TemplateTypeParmType>(Underlying))
+return isDefaultLibrary(TD->getDecl());
+if (auto *TD = Underlying->getAsTagDecl())
+return isDefaultLibrary(TD);
+return false;
 }
 // For a macro usage `DUMP(foo)`, we want:
 //  - DUMP --> "macro"
 //  - foo --> "variable".
 return {};
 // Tokens expanded from macro args are potentially highlightable.
 return getHighlightableSpellingToken(SM.getImmediateSpellingLoc(L), SM);
 }
-unsigned evaluateHighlightPriority(HighlightingKind Kind) {
+unsigned evaluateHighlightPriority(const HighlightingToken &Tok) {
 enum HighlightPriority { Dependent = 0, Resolved = 1 };
-return Kind == HighlightingKind::DependentType ||
+return (Tok.Modifiers & (1 << uint32_t(HighlightingModifier::DependentName)))
-Kind == HighlightingKind::DependentName
 ? Dependent
 : Resolved;
 }
-// Sometimes we get conflicts between findExplicitReferences() returning
+// Sometimes we get multiple tokens at the same location:
-// a heuristic result for a dependent name (e.g. Method) and
+//
-// CollectExtraHighlighting returning a fallback dependent highlighting (e.g.
+// - findExplicitReferences() returns a heuristic result for a dependent name
-// DependentName). In such cases, resolve the conflict in favour of the
+//   (e.g. Method) and CollectExtraHighlighting returning a fallback dependent
-// resolved (non-dependent) highlighting.
+//   highlighting (e.g. Unknown+Dependent).
-// With macros we can get other conflicts (if a spelled token has multiple
+// - macro arguments are expanded multiple times and have different roles
-// expansions with different token types) which we can't usefully resolve.
+// - broken code recovery produces several AST nodes at the same location
+//
+// We should either resolve these to a single token, or drop them all.
+// Our heuristics are:
+//
+// - token kinds that come with "dependent-name" modifiers are less reliable
+//   (these tend to be vague, like Type or Unknown)
+// - if we have multiple equally reliable kinds, drop token rather than guess
+// - take the union of modifiers from all tokens
+//
+// In particular, heuristically resolved dependent names get their heuristic
+// kind, plus the dependent modifier.
 llvm::Optional<HighlightingToken>
 resolveConflict(ArrayRef<HighlightingToken> Tokens) {
 if (Tokens.size() == 1)
 return Tokens[0];
 if (Tokens.size() != 2)
 return llvm::None;
-unsigned Priority1 = evaluateHighlightPriority(Tokens[0].Kind);
+unsigned Priority1 = evaluateHighlightPriority(Tokens[0]);
-unsigned Priority2 = evaluateHighlightPriority(Tokens[1].Kind);
+unsigned Priority2 = evaluateHighlightPriority(Tokens[1]);
-if (Priority1 == Priority2)
+if (Priority1 == Priority2 && Tokens[0].Kind != Tokens[1].Kind)
 return llvm::None;
-return Priority1 > Priority2 ? Tokens[0] : Tokens[1];
+auto Result = Priority1 > Priority2 ? Tokens[0] : Tokens[1];
+Result.Modifiers = Tokens[0].Modifiers | Tokens[1].Modifiers;
+return Result;
 }
 /// Consumes source locations and maps them to text ranges for highlightings.
 class HighlightingsBuilder {
 public:
 HighlightingsBuilder(const ParsedAST &AST)
 : TB(AST.getTokens()), SourceMgr(AST.getSourceManager()),
 LangOpts(AST.getLangOpts()) {}
-void addToken(HighlightingToken T) { Tokens.push_back(T); }
+HighlightingToken &addToken(SourceLocation Loc, HighlightingKind Kind) {
-void addToken(SourceLocation Loc, HighlightingKind Kind) {
 Loc = getHighlightableSpellingToken(Loc, SourceMgr);
 if (Loc.isInvalid())
-return;
+return InvalidHighlightingToken;
 const auto *Tok = TB.spelledTokenAt(Loc);
 assert(Tok);
+return addToken(
-auto Range = halfOpenToRange(SourceMgr,
+halfOpenToRange(SourceMgr,
-Tok->range(SourceMgr).toCharRange(SourceMgr));
+Tok->range(SourceMgr).toCharRange(SourceMgr)),
-Tokens.push_back(HighlightingToken{Kind, std::move(Range)});
+Kind);
+}
+HighlightingToken &addToken(Range R, HighlightingKind Kind) {
+HighlightingToken HT;
+HT.R = std::move(R);
+HT.Kind = Kind;
+Tokens.push_back(std::move(HT));
+return Tokens.back();
 }
 std::vector<HighlightingToken> collect(ParsedAST &AST) && {
 // Initializer lists can give duplicates of tokens, therefore all tokens
 // must be deduplicated.
 NonConflicting.push_back(*Resolved);
 // TokRef[Conflicting.size()] is the next token with a different range (or
 // the end of the Tokens).
 TokRef = TokRef.drop_front(Conflicting.size());
 }
-// Add tokens indicating lines skipped by the preprocessor.
+const auto &SM = AST.getSourceManager();
-for (const Range &R : AST.getMacros().SkippedRanges) {
+StringRef MainCode = SM.getBufferOrFake(SM.getMainFileID()).getBuffer();
+// Merge token stream with "inactive line" markers.
+std::vector<HighlightingToken> WithInactiveLines;
+auto SortedSkippedRanges = AST.getMacros().SkippedRanges;
+llvm::sort(SortedSkippedRanges);
+auto It = NonConflicting.begin();
+for (const Range &R : SortedSkippedRanges) {
 // Create one token for each line in the skipped range, so it works
 // with line-based diffing.
 assert(R.start.line <= R.end.line);
-for (int Line = R.start.line; Line < R.end.line; ++Line) {
+for (int Line = R.start.line; Line <= R.end.line; ++Line) {
-// Don't bother computing the offset for the end of the line, just use
+// If the end of the inactive range is at the beginning
-// zero. The client will treat this highlighting kind specially, and
+// of a line, that line is not inactive.
-// highlight the entire line visually (i.e. not just to where the text
+if (Line == R.end.line && R.end.character == 0)
-// on the line ends, but to the end of the screen).
+continue;
-NonConflicting.push_back({HighlightingKind::InactiveCode,
+// Copy tokens before the inactive line
-{Position{Line, 0}, Position{Line, 0}}});
+for (; It != NonConflicting.end() && It->R.start.line < Line; ++It)
+WithInactiveLines.push_back(std::move(*It));
+// Add a token for the inactive line itself.
+auto StartOfLine = positionToOffset(MainCode, Position{Line, 0});
+if (StartOfLine) {
+StringRef LineText =
+MainCode.drop_front(*StartOfLine).take_until([](char C) {
+return C == '\n';
+});
+HighlightingToken HT;
+WithInactiveLines.emplace_back();
+WithInactiveLines.back().Kind = HighlightingKind::InactiveCode;
+WithInactiveLines.back().R.start.line = Line;
+WithInactiveLines.back().R.end.line = Line;
+WithInactiveLines.back().R.end.character =
+static_cast<int>(lspLength(LineText));
+} else {
+elog("Failed to convert position to offset: {0}",
+StartOfLine.takeError());
+}
+// Skip any other tokens on the inactive line. e.g.
+// `#ifndef Foo` is considered as part of an inactive region when Foo is
+// defined, and there is a Foo macro token.
+// FIXME: we should reduce the scope of the inactive region to not
+// include the directive itself.
+while (It != NonConflicting.end() && It->R.start.line == Line)
+++It;
 }
 }
-// Re-sort the tokens because that's what the diffing expects.
+// Copy tokens after the last inactive line
-llvm::sort(NonConflicting);
+for (; It != NonConflicting.end(); ++It)
-return NonConflicting;
+WithInactiveLines.push_back(std::move(*It));
-}
+return WithInactiveLines;
+}
+const HeuristicResolver *getResolver() const { return Resolver; }
 private:
 const syntax::TokenBuffer &TB;
 const SourceManager &SourceMgr;
 const LangOptions &LangOpts;
 std::vector<HighlightingToken> Tokens;
+const HeuristicResolver *Resolver;
+// returned from addToken(InvalidLoc)
+HighlightingToken InvalidHighlightingToken;
 };
+llvm::Optional<HighlightingModifier> scopeModifier(const NamedDecl *D) {
+const DeclContext *DC = D->getDeclContext();
+// Injected "Foo" within the class "Foo" has file scope, not class scope.
+if (auto *R = dyn_cast_or_null<RecordDecl>(D))
+if (R->isInjectedClassName())
+DC = DC->getParent();
+// Lambda captures are considered function scope, not class scope.
+if (llvm::isa<FieldDecl>(D))
+if (const auto *RD = llvm::dyn_cast<RecordDecl>(DC))
+if (RD->isLambda())
+return HighlightingModifier::FunctionScope;
+// Walk up the DeclContext hierarchy until we find something interesting.
+for (; !DC->isFileContext(); DC = DC->getParent()) {
+if (DC->isFunctionOrMethod())
+return HighlightingModifier::FunctionScope;
+if (DC->isRecord())
+return HighlightingModifier::ClassScope;
+}
+// Some template parameters (e.g. those for variable templates) don't have
+// meaningful DeclContexts. That doesn't mean they're global!
+if (DC->isTranslationUnit() && D->isTemplateParameter())
+return llvm::None;
+// ExternalLinkage threshold could be tweaked, e.g. module-visible as global.
+if (D->getLinkageInternal() < ExternalLinkage)
+return HighlightingModifier::FileScope;
+return HighlightingModifier::GlobalScope;
+}
+llvm::Optional<HighlightingModifier> scopeModifier(const Type *T) {
+if (!T)
+return llvm::None;
+if (T->isBuiltinType())
+return HighlightingModifier::GlobalScope;
+if (auto *TD = dyn_cast<TemplateTypeParmType>(T))
+return scopeModifier(TD->getDecl());
+if (auto *TD = T->getAsTagDecl())
+return scopeModifier(TD);
+return llvm::None;
+}
 /// Produces highlightings, which are not captured by findExplicitReferences,
 /// e.g. highlights dependent names and 'auto' as the underlying type.
 class CollectExtraHighlightings
 : public RecursiveASTVisitor<CollectExtraHighlightings> {
 public:
 CollectExtraHighlightings(HighlightingsBuilder &H) : H(H) {}
 bool VisitDecltypeTypeLoc(DecltypeTypeLoc L) {
-if (auto K = kindForType(L.getTypePtr()))
+if (auto K = kindForType(L.getTypePtr(), H.getResolver())) {
-H.addToken(L.getBeginLoc(), *K);
+auto &Tok = H.addToken(L.getBeginLoc(), *K)
+.addModifier(HighlightingModifier::Deduced);
+if (auto Mod = scopeModifier(L.getTypePtr()))
+Tok.addModifier(*Mod);
+if (isDefaultLibrary(L.getTypePtr()))
+Tok.addModifier(HighlightingModifier::DefaultLibrary);
+}
 return true;
 }
 bool VisitDeclaratorDecl(DeclaratorDecl *D) {
 auto *AT = D->getType()->getContainedAutoType();
 if (!AT)
 return true;
-if (auto K = kindForType(AT->getDeducedType().getTypePtrOrNull()))
+if (auto K = kindForType(AT->getDeducedType().getTypePtrOrNull(),
-H.addToken(D->getTypeSpecStartLoc(), *K);
+H.getResolver())) {
+auto &Tok = H.addToken(D->getTypeSpecStartLoc(), *K)
+.addModifier(HighlightingModifier::Deduced);
+const Type *Deduced = AT->getDeducedType().getTypePtrOrNull();
+if (auto Mod = scopeModifier(Deduced))
+Tok.addModifier(*Mod);
+if (isDefaultLibrary(Deduced))
+Tok.addModifier(HighlightingModifier::DefaultLibrary);
+}
+return true;
+}
+// We handle objective-C selectors specially, because one reference can
+// cover several non-contiguous tokens.
+void highlightObjCSelector(const ArrayRef<SourceLocation> &Locs, bool Decl,
+bool Class, bool DefaultLibrary) {
+HighlightingKind Kind =
+Class ? HighlightingKind::StaticMethod : HighlightingKind::Method;
+for (SourceLocation Part : Locs) {
+auto &Tok =
+H.addToken(Part, Kind).addModifier(HighlightingModifier::ClassScope);
+if (Decl)
+Tok.addModifier(HighlightingModifier::Declaration);
+if (Class)
+Tok.addModifier(HighlightingModifier::Static);
+if (DefaultLibrary)
+Tok.addModifier(HighlightingModifier::DefaultLibrary);
+}
+}
+bool VisitObjCMethodDecl(ObjCMethodDecl *OMD) {
+llvm::SmallVector<SourceLocation> Locs;
+OMD->getSelectorLocs(Locs);
+highlightObjCSelector(Locs, /*Decl=*/true, OMD->isClassMethod(),
+isDefaultLibrary(OMD));
+return true;
+}
+bool VisitObjCMessageExpr(ObjCMessageExpr *OME) {
+llvm::SmallVector<SourceLocation> Locs;
+OME->getSelectorLocs(Locs);
+bool DefaultLibrary = false;
+if (ObjCMethodDecl *OMD = OME->getMethodDecl())
+DefaultLibrary = isDefaultLibrary(OMD);
+highlightObjCSelector(Locs, /*Decl=*/false, OME->isClassMessage(),
+DefaultLibrary);
 return true;
 }
 bool VisitOverloadExpr(OverloadExpr *E) {
 if (!E->decls().empty())
 return true; // handled by findExplicitReferences.
-H.addToken(E->getNameLoc(), HighlightingKind::DependentName);
+auto &Tok = H.addToken(E->getNameLoc(), HighlightingKind::Unknown)
+.addModifier(HighlightingModifier::DependentName);
+if (llvm::isa<UnresolvedMemberExpr>(E))
+Tok.addModifier(HighlightingModifier::ClassScope);
+// other case is UnresolvedLookupExpr, scope is unknown.
 return true;
 }
 bool VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E) {
-H.addToken(E->getMemberNameInfo().getLoc(),
+H.addToken(E->getMemberNameInfo().getLoc(), HighlightingKind::Unknown)
-HighlightingKind::DependentName);
+.addModifier(HighlightingModifier::DependentName)
+.addModifier(HighlightingModifier::ClassScope);
 return true;
 }
 bool VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) {
-H.addToken(E->getNameInfo().getLoc(), HighlightingKind::DependentName);
+H.addToken(E->getNameInfo().getLoc(), HighlightingKind::Unknown)
+.addModifier(HighlightingModifier::DependentName)
+.addModifier(HighlightingModifier::ClassScope);
 return true;
 }
 bool VisitDependentNameTypeLoc(DependentNameTypeLoc L) {
-H.addToken(L.getNameLoc(), HighlightingKind::DependentType);
+H.addToken(L.getNameLoc(), HighlightingKind::Type)
+.addModifier(HighlightingModifier::DependentName)
+.addModifier(HighlightingModifier::ClassScope);
 return true;
 }
 bool VisitDependentTemplateSpecializationTypeLoc(
 DependentTemplateSpecializationTypeLoc L) {
-H.addToken(L.getTemplateNameLoc(), HighlightingKind::DependentType);
+H.addToken(L.getTemplateNameLoc(), HighlightingKind::Type)
-return true;
+.addModifier(HighlightingModifier::DependentName)
+.addModifier(HighlightingModifier::ClassScope);
+return true;
+}
+bool TraverseTemplateArgumentLoc(TemplateArgumentLoc L) {
+// Handle template template arguments only (other arguments are handled by
+// their Expr, TypeLoc etc values).
+if (L.getArgument().getKind() != TemplateArgument::Template &&
+L.getArgument().getKind() != TemplateArgument::TemplateExpansion)
+return RecursiveASTVisitor::TraverseTemplateArgumentLoc(L);
+TemplateName N = L.getArgument().getAsTemplateOrTemplatePattern();
+switch (N.getKind()) {
+case TemplateName::OverloadedTemplate:
+// Template template params must always be class templates.
+// Don't bother to try to work out the scope here.
+H.addToken(L.getTemplateNameLoc(), HighlightingKind::Class);
+break;
+case TemplateName::DependentTemplate:
+case TemplateName::AssumedTemplate:
+H.addToken(L.getTemplateNameLoc(), HighlightingKind::Class)
+.addModifier(HighlightingModifier::DependentName);
+break;
+case TemplateName::Template:
+case TemplateName::QualifiedTemplate:
+case TemplateName::SubstTemplateTemplateParm:
+case TemplateName::SubstTemplateTemplateParmPack:
+// Names that could be resolved to a TemplateDecl are handled elsewhere.
+break;
+}
+return RecursiveASTVisitor::TraverseTemplateArgumentLoc(L);
 }
 // findExplicitReferences will walk nested-name-specifiers and
 // find anything that can be resolved to a Decl. However, non-leaf
 // components of nested-name-specifiers which are dependent names
 // (kind "Identifier") cannot be resolved to a decl, so we visit
 // them here.
 bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc Q) {
 if (NestedNameSpecifier *NNS = Q.getNestedNameSpecifier()) {
 if (NNS->getKind() == NestedNameSpecifier::Identifier)
-H.addToken(Q.getLocalBeginLoc(), HighlightingKind::DependentType);
+H.addToken(Q.getLocalBeginLoc(), HighlightingKind::Type)
+.addModifier(HighlightingModifier::DependentName)
+.addModifier(HighlightingModifier::ClassScope);
 }
 return RecursiveASTVisitor::TraverseNestedNameSpecifierLoc(Q);
 }
 private:
 HighlightingsBuilder &H;
 };
-void write32be(uint32_t I, llvm::raw_ostream &OS) {
-std::array<char, 4> Buf;
-llvm::support::endian::write32be(Buf.data(), I);
-OS.write(Buf.data(), Buf.size());
-}
-void write16be(uint16_t I, llvm::raw_ostream &OS) {
-std::array<char, 2> Buf;
-llvm::support::endian::write16be(Buf.data(), I);
-OS.write(Buf.data(), Buf.size());
-}
-// Get the highlightings on \c Line where the first entry of line is at \c
-// StartLineIt. If it is not at \c StartLineIt an empty vector is returned.
-ArrayRef<HighlightingToken>
-takeLine(ArrayRef<HighlightingToken> AllTokens,
-ArrayRef<HighlightingToken>::iterator StartLineIt, int Line) {
-return ArrayRef<HighlightingToken>(StartLineIt, AllTokens.end())
-.take_while([Line](const HighlightingToken &Token) {
-return Token.R.start.line == Line;
-});
-}
 } // namespace
 std::vector<HighlightingToken> getSemanticHighlightings(ParsedAST &AST) {
 auto &C = AST.getASTContext();
 // Add highlightings for AST nodes.
 HighlightingsBuilder Builder(AST);
 // Highlight 'decltype' and 'auto' as their underlying types.
 CollectExtraHighlightings(Builder).TraverseAST(C);
 // Highlight all decls and references coming from the AST.
-findExplicitReferences(C, [&](ReferenceLoc R) {
+findExplicitReferences(
-if (auto Kind = kindForReference(R))
+C,
-Builder.addToken(R.NameLoc, *Kind);
+[&](ReferenceLoc R) {
-});
+for (const NamedDecl *Decl : R.Targets) {
+if (!canHighlightName(Decl->getDeclName()))
+continue;
+auto Kind = kindForDecl(Decl, AST.getHeuristicResolver());
+if (!Kind)
+continue;
+auto &Tok = Builder.addToken(R.NameLoc, *Kind);
+// The attribute tests don't want to look at the template.
+if (auto *TD = dyn_cast<TemplateDecl>(Decl)) {
+if (auto *Templated = TD->getTemplatedDecl())
+Decl = Templated;
+}
+if (auto Mod = scopeModifier(Decl))
+Tok.addModifier(*Mod);
+if (isConst(Decl))
+Tok.addModifier(HighlightingModifier::Readonly);
+if (isStatic(Decl))
+Tok.addModifier(HighlightingModifier::Static);
+if (isAbstract(Decl))
+Tok.addModifier(HighlightingModifier::Abstract);
+if (isDependent(Decl))
+Tok.addModifier(HighlightingModifier::DependentName);
+if (isDefaultLibrary(Decl))
+Tok.addModifier(HighlightingModifier::DefaultLibrary);
+if (Decl->isDeprecated())
+Tok.addModifier(HighlightingModifier::Deprecated);
+// Do not treat an UnresolvedUsingValueDecl as a declaration.
+// It's more common to think of it as a reference to the
+// underlying declaration.
+if (R.IsDecl && !isa<UnresolvedUsingValueDecl>(Decl))
+Tok.addModifier(HighlightingModifier::Declaration);
+}
+},
+AST.getHeuristicResolver());
 // Add highlightings for macro references.
-for (const auto &SIDToRefs : AST.getMacros().MacroRefs) {
+auto AddMacro = [&](const MacroOccurrence &M) {
+auto &T = Builder.addToken(M.Rng, HighlightingKind::Macro);
+T.addModifier(HighlightingModifier::GlobalScope);
+if (M.IsDefinition)
+T.addModifier(HighlightingModifier::Declaration);
+};
+for (const auto &SIDToRefs : AST.getMacros().MacroRefs)
 for (const auto &M : SIDToRefs.second)
-Builder.addToken({HighlightingKind::Macro, M});
+AddMacro(M);
-}
 for (const auto &M : AST.getMacros().UnknownMacros)
-Builder.addToken({HighlightingKind::Macro, M});
+AddMacro(M);
 return std::move(Builder).collect(AST);
 }
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingKind K) {
 return OS << "Field";
 case HighlightingKind::StaticField:
 return OS << "StaticField";
 case HighlightingKind::Class:
 return OS << "Class";
+case HighlightingKind::Interface:
+return OS << "Interface";
 case HighlightingKind::Enum:
 return OS << "Enum";
 case HighlightingKind::EnumConstant:
 return OS << "EnumConstant";
 case HighlightingKind::Typedef:
 return OS << "Typedef";
-case HighlightingKind::DependentType:
+case HighlightingKind::Type:
-return OS << "DependentType";
+return OS << "Type";
-case HighlightingKind::DependentName:
+case HighlightingKind::Unknown:
-return OS << "DependentName";
+return OS << "Unknown";
 case HighlightingKind::Namespace:
 return OS << "Namespace";
 case HighlightingKind::TemplateParameter:
 return OS << "TemplateParameter";
 case HighlightingKind::Concept:
 case HighlightingKind::InactiveCode:
 return OS << "InactiveCode";
 }
 llvm_unreachable("invalid HighlightingKind");
 }
+llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingModifier K) {
-std::vector<LineHighlightings>
+switch (K) {
-diffHighlightings(ArrayRef<HighlightingToken> New,
+case HighlightingModifier::Declaration:
-ArrayRef<HighlightingToken> Old) {
+return OS << "decl"; // abbrevation for common case
-assert(std::is_sorted(New.begin(), New.end()) &&
+default:
-"New must be a sorted vector");
+return OS << toSemanticTokenModifier(K);
-assert(std::is_sorted(Old.begin(), Old.end()) &&
+}
-"Old must be a sorted vector");
-// FIXME: There's an edge case when tokens span multiple lines. If the first
-// token on the line started on a line above the current one and the rest of
-// the line is the equal to the previous one than we will remove all
-// highlights but the ones for the token spanning multiple lines. This means
-// that when we get into the LSP layer the only highlights that will be
-// visible are the ones for the token spanning multiple lines.
-// Example:
-// EndOfMultilineToken  Token Token Token
-// If "Token Token Token" don't differ from previously the line is
-// incorrectly removed. Suggestion to fix is to separate any multiline tokens
-// into one token for every line it covers. This requires reading from the
-// file buffer to figure out the length of each line though.
-std::vector<LineHighlightings> DiffedLines;
-// ArrayRefs to the current line in the highlightings.
-ArrayRef<HighlightingToken> NewLine(New.begin(),
-/*length*/ static_cast<size_t>(0));
-ArrayRef<HighlightingToken> OldLine(Old.begin(),
-/*length*/ static_cast<size_t>(0));
-auto NewEnd = New.end();
-auto OldEnd = Old.end();
-auto NextLineNumber = [&]() {
-int NextNew = NewLine.end() != NewEnd ? NewLine.end()->R.start.line
-: std::numeric_limits<int>::max();
-int NextOld = OldLine.end() != OldEnd ? OldLine.end()->R.start.line
-: std::numeric_limits<int>::max();
-return std::min(NextNew, NextOld);
-};
-for (int LineNumber = 0; NewLine.end() < NewEnd || OldLine.end() < OldEnd;
-LineNumber = NextLineNumber()) {
-NewLine = takeLine(New, NewLine.end(), LineNumber);
-OldLine = takeLine(Old, OldLine.end(), LineNumber);
-if (NewLine != OldLine) {
-DiffedLines.push_back({LineNumber, NewLine, /*IsInactive=*/false});
-// Turn a HighlightingKind::InactiveCode token into the IsInactive flag.
-auto &AddedLine = DiffedLines.back();
-llvm::erase_if(AddedLine.Tokens, [&](const HighlightingToken &T) {
-if (T.Kind == HighlightingKind::InactiveCode) {
-AddedLine.IsInactive = true;
-return true;
-}
-return false;
-});
-}
-}
-return DiffedLines;
 }
 bool operator==(const HighlightingToken &L, const HighlightingToken &R) {
-return std::tie(L.R, L.Kind) == std::tie(R.R, R.Kind);
+return std::tie(L.R, L.Kind, L.Modifiers) ==
+std::tie(R.R, R.Kind, R.Modifiers);
 }
 bool operator<(const HighlightingToken &L, const HighlightingToken &R) {
-return std::tie(L.R, L.Kind) < std::tie(R.R, R.Kind);
+return std::tie(L.R, L.Kind, R.Modifiers) <
-}
+std::tie(R.R, R.Kind, R.Modifiers);
-bool operator==(const LineHighlightings &L, const LineHighlightings &R) {
-return std::tie(L.Line, L.Tokens) == std::tie(R.Line, R.Tokens);
 }
 std::vector<SemanticToken>
 toSemanticTokens(llvm::ArrayRef<HighlightingToken> Tokens) {
 assert(std::is_sorted(Tokens.begin(), Tokens.end()));
 std::vector<SemanticToken> Result;
 const HighlightingToken *Last = nullptr;
 for (const HighlightingToken &Tok : Tokens) {
-// FIXME: support inactive code - we need to provide the actual bounds.
-if (Tok.Kind == HighlightingKind::InactiveCode)
-continue;
 Result.emplace_back();
 SemanticToken &Out = Result.back();
 // deltaStart/deltaLine are relative if possible.
 if (Last) {
 assert(Tok.R.start.line >= Last->R.start.line);
 Out.deltaStart = Tok.R.start.character;
 }
 assert(Tok.R.end.line == Tok.R.start.line);
 Out.length = Tok.R.end.character - Tok.R.start.character;
 Out.tokenType = static_cast<unsigned>(Tok.Kind);
+Out.tokenModifiers = Tok.Modifiers;
 Last = &Tok;
 }
 return Result;
 }
 case HighlightingKind::Parameter:
 return "parameter";
 case HighlightingKind::Function:
 return "function";
 case HighlightingKind::Method:
-return "member";
+return "method";
 case HighlightingKind::StaticMethod:
-// FIXME: better function/member with static modifier?
+// FIXME: better method with static modifier?
 return "function";
 case HighlightingKind::Field:
-return "member";
+return "property";
 case HighlightingKind::Class:
 return "class";
+case HighlightingKind::Interface:
+return "interface";
 case HighlightingKind::Enum:
 return "enum";
 case HighlightingKind::EnumConstant:
-return "enumConstant"; // nonstandard
+return "enumMember";
 case HighlightingKind::Typedef:
+case HighlightingKind::Type:
 return "type";
-case HighlightingKind::DependentType:
+case HighlightingKind::Unknown:
-return "dependent"; // nonstandard
+return "unknown"; // nonstandard
-case HighlightingKind::DependentName:
-return "dependent"; // nonstandard
 case HighlightingKind::Namespace:
 return "namespace";
 case HighlightingKind::TemplateParameter:
 return "typeParameter";
 case HighlightingKind::Concept:
 return "comment";
 }
 llvm_unreachable("unhandled HighlightingKind");
 }
-std::vector<TheiaSemanticHighlightingInformation>
+llvm::StringRef toSemanticTokenModifier(HighlightingModifier Modifier) {
-toTheiaSemanticHighlightingInformation(
+switch (Modifier) {
-llvm::ArrayRef<LineHighlightings> Tokens) {
+case HighlightingModifier::Declaration:
-if (Tokens.size() == 0)
+return "declaration";
-return {};
+case HighlightingModifier::Deprecated:
+return "deprecated";
-// FIXME: Tokens might be multiple lines long (block comments) in this case
+case HighlightingModifier::Readonly:
-// this needs to add multiple lines for those tokens.
+return "readonly";
-std::vector<TheiaSemanticHighlightingInformation> Lines;
+case HighlightingModifier::Static:
-Lines.reserve(Tokens.size());
+return "static";
-for (const auto &Line : Tokens) {
+case HighlightingModifier::Deduced:
-llvm::SmallVector<char, 128> LineByteTokens;
+return "deduced"; // nonstandard
-llvm::raw_svector_ostream OS(LineByteTokens);
+case HighlightingModifier::Abstract:
-for (const auto &Token : Line.Tokens) {
+return "abstract";
-// Writes the token to LineByteTokens in the byte format specified by the
+case HighlightingModifier::DependentName:
-// LSP proposal. Described below.
+return "dependentName"; // nonstandard
-// |<---- 4 bytes ---->|<-- 2 bytes -->|<--- 2 bytes -->|
+case HighlightingModifier::DefaultLibrary:
-// |    character      |  length       |    index       |
+return "defaultLibrary";
+case HighlightingModifier::FunctionScope:
-write32be(Token.R.start.character, OS);
+return "functionScope"; // nonstandard
-write16be(Token.R.end.character - Token.R.start.character, OS);
+case HighlightingModifier::ClassScope:
-write16be(static_cast<int>(Token.Kind), OS);
+return "classScope"; // nonstandard
-}
+case HighlightingModifier::FileScope:
+return "fileScope"; // nonstandard
-Lines.push_back({Line.Line, encodeBase64(LineByteTokens), Line.IsInactive});
+case HighlightingModifier::GlobalScope:
-}
+return "globalScope"; // nonstandard
+}
-return Lines;
+llvm_unreachable("unhandled HighlightingModifier");
-}
-llvm::StringRef toTextMateScope(HighlightingKind Kind) {
-// FIXME: Add scopes for C and Objective C.
-switch (Kind) {
-case HighlightingKind::Function:
-return "entity.name.function.cpp";
-case HighlightingKind::Method:
-return "entity.name.function.method.cpp";
-case HighlightingKind::StaticMethod:
-return "entity.name.function.method.static.cpp";
-case HighlightingKind::Variable:
-return "variable.other.cpp";
-case HighlightingKind::LocalVariable:
-return "variable.other.local.cpp";
-case HighlightingKind::Parameter:
-return "variable.parameter.cpp";
-case HighlightingKind::Field:
-return "variable.other.field.cpp";
-case HighlightingKind::StaticField:
-return "variable.other.field.static.cpp";
-case HighlightingKind::Class:
-return "entity.name.type.class.cpp";
-case HighlightingKind::Enum:
-return "entity.name.type.enum.cpp";
-case HighlightingKind::EnumConstant:
-return "variable.other.enummember.cpp";
-case HighlightingKind::Typedef:
-return "entity.name.type.typedef.cpp";
-case HighlightingKind::DependentType:
-return "entity.name.type.dependent.cpp";
-case HighlightingKind::DependentName:
-return "entity.name.other.dependent.cpp";
-case HighlightingKind::Namespace:
-return "entity.name.namespace.cpp";
-case HighlightingKind::TemplateParameter:
-return "entity.name.type.template.cpp";
-case HighlightingKind::Concept:
-return "entity.name.type.concept.cpp";
-case HighlightingKind::Primitive:
-return "storage.type.primitive.cpp";
-case HighlightingKind::Macro:
-return "entity.name.function.preprocessor.cpp";
-case HighlightingKind::InactiveCode:
-return "meta.disabled";
-}
-llvm_unreachable("unhandled HighlightingKind");
 }
 std::vector<SemanticTokensEdit>
 diffTokens(llvm::ArrayRef<SemanticToken> Old,
 llvm::ArrayRef<SemanticToken> New) {

Mercurial > hg > CbC > CbC_llvm

comparison clang-tools-extra/clangd/SemanticHighlighting.cpp @ 221:79ff65ed7e25