[PATCH] D63127: [clang-tidy] Fixed checker for abseil to work in C++17 mode

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom created this revision.
jvikstrom added reviewers: hokein, gribozavr.
Herald added a subscriber: xazax.hun.
Herald added a project: clang.

Fixes the checker for abseil to make tests pass in C++17 mode


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D63127

Files:
  clang-tools-extra/clang-tidy/abseil/DurationUnnecessaryConversionCheck.cpp
  clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.cpp
  clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp
  clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp
  clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.h
  clang-tools-extra/test/clang-tidy/abseil-duration-unnecessary-conversion.cpp
  clang-tools-extra/test/clang-tidy/abseil-faster-strsplit-delimiter.cpp
  clang-tools-extra/test/clang-tidy/abseil-time-subtraction.cpp
  clang-tools-extra/test/clang-tidy/abseil-upgrade-duration-conversions.cpp

Index: clang-tools-extra/test/clang-tidy/abseil-upgrade-duration-conversions.cpp
===
--- clang-tools-extra/test/clang-tidy/abseil-upgrade-duration-conversions.cpp
+++ clang-tools-extra/test/clang-tidy/abseil-upgrade-duration-conversions.cpp
@@ -1,5 +1,4 @@
-// RUN: %check_clang_tidy -std=c++11,c++14 %s abseil-upgrade-duration-conversions %t -- -- -I%S/Inputs
-// FIXME: Fix the checker to work in C++17 mode.
+// RUN: %check_clang_tidy -std=c++11,c++14,c++17 %s abseil-upgrade-duration-conversions %t -- -- -I%S/Inputs
 
 using int64_t = long long;
 
Index: clang-tools-extra/test/clang-tidy/abseil-time-subtraction.cpp
===
--- clang-tools-extra/test/clang-tidy/abseil-time-subtraction.cpp
+++ clang-tools-extra/test/clang-tidy/abseil-time-subtraction.cpp
@@ -1,5 +1,4 @@
-// RUN: %check_clang_tidy -std=c++11,c++14 %s abseil-time-subtraction %t -- -- -I %S/Inputs
-// FIXME: Fix the checker to work in C++17 mode.
+// RUN: %check_clang_tidy -std=c++11,c++14,c++17 %s abseil-time-subtraction %t -- -- -I %S/Inputs
 
 #include "absl/time/time.h"
 
Index: clang-tools-extra/test/clang-tidy/abseil-faster-strsplit-delimiter.cpp
===
--- clang-tools-extra/test/clang-tidy/abseil-faster-strsplit-delimiter.cpp
+++ clang-tools-extra/test/clang-tidy/abseil-faster-strsplit-delimiter.cpp
@@ -1,5 +1,4 @@
-// RUN: %check_clang_tidy -std=c++11,c++14 %s abseil-faster-strsplit-delimiter %t
-// FIXME: Fix the checker to work in C++17 mode.
+// RUN: %check_clang_tidy -std=c++11,c++14,c++17 %s abseil-faster-strsplit-delimiter %t
 
 namespace absl {
 
Index: clang-tools-extra/test/clang-tidy/abseil-duration-unnecessary-conversion.cpp
===
--- clang-tools-extra/test/clang-tidy/abseil-duration-unnecessary-conversion.cpp
+++ clang-tools-extra/test/clang-tidy/abseil-duration-unnecessary-conversion.cpp
@@ -1,5 +1,4 @@
-// RUN: %check_clang_tidy -std=c++11,c++14 %s abseil-duration-unnecessary-conversion %t -- -- -I %S/Inputs
-// FIXME: Fix the checker to work in C++17 mode.
+// RUN: %check_clang_tidy -std=c++11,c++14,c++17 %s abseil-duration-unnecessary-conversion %t -- -- -I %S/Inputs
 
 #include "absl/time/time.h"
 
Index: clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.h
===
--- clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.h
+++ clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.h
@@ -17,7 +17,8 @@
 namespace tidy {
 namespace abseil {
 
-/// Finds deprecated uses of `absl::Duration` arithmetic operators and factories.
+/// Finds deprecated uses of `absl::Duration` arithmetic operators and
+/// factories.
 ///
 /// For the user-facing documentation see:
 /// http://clang.llvm.org/extra/clang-tidy/checks/abseil-upgrade-duration-conversions.html
Index: clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp
===
--- clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp
+++ clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp
@@ -10,6 +10,8 @@
 #include "DurationRewriter.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/Basic/Specifiers.h"
 
 using namespace clang::ast_matchers;
 
@@ -34,13 +36,15 @@
   Finder->addMatcher(
   cxxOperatorCallExpr(
   argumentCountIs(2),
-  hasArgument(
-  0, expr(hasType(cxxRecordDecl(hasName("::absl::Duration"),
+  hasArgument(0,
+  expr(hasType(cxxRecordDecl(hasName("::absl::Duration"
+  .bind("arg0")),
   hasArgument(1, expr().bind("arg")),
   callee(functionDecl(
   hasParent(functionTemplat

[PATCH] D63149: Added AST matcher for ignoring elidable constructors

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom created this revision.
jvikstrom added reviewers: hokein, gribozavr.
Herald added a project: clang.
Herald added a subscriber: cfe-commits.

Added AST matcher for ignoring elidable move constructors


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D63149

Files:
  clang/docs/LibASTMatchersReference.html
  clang/include/clang/ASTMatchers/ASTMatchers.h
  clang/lib/ASTMatchers/Dynamic/Registry.cpp
  clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp

Index: clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
===
--- clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -566,6 +566,71 @@
llvm::make_unique>("x")));
 }
 
+TEST(Matcher, IgnoresElidableConstructors) {
+  StatementMatcher matcher = cxxOperatorCallExpr(
+hasArgument(1, callExpr(hasArgument(0, 
+ignoringElidableMoveConstructorCall(
+  ignoringParenImpCasts(expr().bind("c")));
+
+  EXPECT_TRUE(matchAndVerifyResultTrue(
+"struct H {};"
+"template H B(T A);"
+"void f(){"
+"H D1;"
+"D1=B(B(1));"
+"}"
+, matcher, llvm::make_unique>("c")));
+
+  EXPECT_TRUE(matchAndVerifyResultTrue(
+"struct H {};"
+"template H B(T A);"
+"void f(){"
+"H D1;"
+"D1=B(1);"
+"}"
+, matcher, llvm::make_unique>("c")));
+}
+
+TEST(Matcher, IgnoresElidableInReturn) {
+  auto matcher = expr(ignoringElidableMoveConstructorCall(declRefExpr()));
+
+  EXPECT_TRUE(matches(
+"struct H{};"
+"H f(){"
+"H g;"
+"return g;"
+"}"
+, matcher));
+
+  EXPECT_FALSE(matches(
+"struct H{};"
+"H f(){"
+"return H();"
+"}"
+, matcher
+  ));
+}
+
+TEST(Matcher, IgnoreElidableConstructorDoesNotMatchConstructors) {
+  auto matcher = varDecl(hasInitializer(ignoringElidableMoveConstructorCall(cxxConstructExpr(;
+  EXPECT_TRUE(matches(
+"struct H {};"
+"void f(){"
+"H D;"
+"}"
+  , matcher));
+
+};
+
+TEST(Matcher, IgnoresElidableDoesNotPreventMatches) {
+  auto matcher = expr(ignoringElidableMoveConstructorCall(integerLiteral()));
+  EXPECT_TRUE(matches(
+"void f(){"
+"int D = 10;"
+"}"
+  , matcher));
+}
+
 TEST(Matcher, BindTheSameNameInAlternatives) {
   StatementMatcher matcher = anyOf(
 binaryOperator(hasOperatorName("+"),
Index: clang/lib/ASTMatchers/Dynamic/Registry.cpp
===
--- clang/lib/ASTMatchers/Dynamic/Registry.cpp
+++ clang/lib/ASTMatchers/Dynamic/Registry.cpp
@@ -320,6 +320,7 @@
   REGISTER_MATCHER(hasUnqualifiedDesugaredType);
   REGISTER_MATCHER(hasValueType);
   REGISTER_MATCHER(ifStmt);
+  REGISTER_MATCHER(ignoringElidableMoveConstructorCall);
   REGISTER_MATCHER(ignoringImpCasts);
   REGISTER_MATCHER(ignoringImplicit);
   REGISTER_MATCHER(ignoringParenCasts);
Index: clang/include/clang/ASTMatchers/ASTMatchers.h
===
--- clang/include/clang/ASTMatchers/ASTMatchers.h
+++ clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -6452,6 +6452,31 @@
   return false;
 }
 
+/// Matches expressions that match InnerMatcher after any elidable constructor are stripped off.
+///
+/// Example matches the entire D1 = ... (matcher = cxxOperatorCallExpr(hasArgument(1, callExpr(hasArgument(0, ignoringElidableMoveConstructorCall(ignoringParenImpCasts(callExpr(
+/// \code
+/// struct H {};
+/// template H B(T A);
+/// void f() {
+///   H D1;
+///   D1 = B(B(1));
+/// }
+/// \endcode
+AST_MATCHER_P(Expr, ignoringElidableMoveConstructorCall,
+  ast_matchers::internal::Matcher, InnerMatcher) {
+  if (const auto* cxx_construct_expr = dyn_cast(&Node)) {
+if (cxx_construct_expr->isElidable()) {
+  if (const auto* materialize_temp = dyn_cast(
+  cxx_construct_expr->getArg(0))) {
+return InnerMatcher.matches(*materialize_temp, Finder, Builder);
+  }
+  return InnerMatcher.matches(*cxx_construct_expr, Finder, Builder);
+}
+  }
+  return InnerMatcher.matches(Node, Finder, Builder);
+}
+
 ////
 // OpenMP handling.
 ////
Index: clang/docs/LibASTMatchersReference.html
===
--- clang/docs/LibASTMatchersReference.html
+++ clang/docs/LibASTMatchersReference.html
@@ -5728,6 +5728,19 @@
 
 
 
+MatcherExpr>ignoringElidableMoveConstructorCallast_matchers::MatcherExpr> InnerMatcher
+Matches expressions that match InnerMatcher after any elidable constructor are stripped off.
+
+Example matches the entire

[PATCH] D63149: Added AST matcher for ignoring elidable constructors

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 204229.
jvikstrom added a comment.

- Made ignoreElidable also ignore materializeTemporaryExpr and reformatted code


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D63149/new/

https://reviews.llvm.org/D63149

Files:
  clang/docs/LibASTMatchersReference.html
  clang/include/clang/ASTMatchers/ASTMatchers.h
  clang/lib/ASTMatchers/Dynamic/Registry.cpp
  clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp

Index: clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
===
--- clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -566,6 +566,71 @@
llvm::make_unique>("x")));
 }
 
+TEST(Matcher, IgnoresElidableConstructors) {
+  StatementMatcher matcher1 = cxxOperatorCallExpr(hasArgument(
+  1, callExpr(hasArgument(
+ 0, ignoringElidableMoveConstructorCall(callExpr().bind("c"));
+  StatementMatcher matcher2 = cxxOperatorCallExpr(hasArgument(
+  1, callExpr(hasArgument(0, ignoringElidableMoveConstructorCall(
+ integerLiteral().bind("c"));
+
+  auto code1 = "struct H {};"
+   "template H B(T A);"
+   "void f(){"
+   "H D1;"
+   "D1=B(B(1));"
+   "}";
+  auto code2 = "struct H {};"
+   "template H B(T A);"
+   "void f(){"
+   "H D1;"
+   "D1=B(1);"
+   "}";
+
+  EXPECT_TRUE(matchesConditionally(code1, matcher1, true, "-std=c++14"));
+  EXPECT_TRUE(matchesConditionally(code1, matcher1, true, "-std=c++17"));
+
+  EXPECT_TRUE(matchesConditionally(code2, matcher2, true, "-std=c++14"));
+  EXPECT_TRUE(matchesConditionally(code2, matcher2, true, "-std=c++17"));
+}
+
+TEST(Matcher, IgnoresElidableInReturn) {
+  auto matcher = expr(ignoringElidableMoveConstructorCall(declRefExpr()));
+  auto code1 = "struct H{};"
+   "H f(){"
+   "H g;"
+   "return g;"
+   "}";
+  auto code2 = "struct H{};"
+   "H f(){"
+   "return H();"
+   "}";
+  EXPECT_TRUE(matchesConditionally(code1, matcher, true, "-std=c++14"));
+  EXPECT_TRUE(matchesConditionally(code1, matcher, true, "-std=c++17"));
+  EXPECT_TRUE(matchesConditionally(code2, matcher, false, "-std=c++14"));
+  EXPECT_TRUE(matchesConditionally(code2, matcher, false, "-std=c++17"));
+}
+
+TEST(Matcher, IgnoreElidableConstructorDoesNotMatchConstructors) {
+  auto matcher = varDecl(
+  hasInitializer(ignoringElidableMoveConstructorCall(cxxConstructExpr(;
+  auto code = "struct H {};"
+  "void f(){"
+  "H D;"
+  "}";
+  EXPECT_TRUE(matchesConditionally(code, matcher, true, "-std=c++14"));
+  EXPECT_TRUE(matchesConditionally(code, matcher, true, "-std=c++17"));
+};
+
+TEST(Matcher, IgnoresElidableDoesNotPreventMatches) {
+  auto matcher = expr(ignoringElidableMoveConstructorCall(integerLiteral()));
+  auto code = "void f(){"
+  "int D = 10;"
+  "}";
+  EXPECT_TRUE(matchesConditionally(code, matcher, true, "-std=c++14"));
+  EXPECT_TRUE(matchesConditionally(code, matcher, true, "-std=c++17"));
+}
+
 TEST(Matcher, BindTheSameNameInAlternatives) {
   StatementMatcher matcher = anyOf(
 binaryOperator(hasOperatorName("+"),
Index: clang/lib/ASTMatchers/Dynamic/Registry.cpp
===
--- clang/lib/ASTMatchers/Dynamic/Registry.cpp
+++ clang/lib/ASTMatchers/Dynamic/Registry.cpp
@@ -320,6 +320,7 @@
   REGISTER_MATCHER(hasUnqualifiedDesugaredType);
   REGISTER_MATCHER(hasValueType);
   REGISTER_MATCHER(ifStmt);
+  REGISTER_MATCHER(ignoringElidableMoveConstructorCall);
   REGISTER_MATCHER(ignoringImpCasts);
   REGISTER_MATCHER(ignoringImplicit);
   REGISTER_MATCHER(ignoringParenCasts);
Index: clang/include/clang/ASTMatchers/ASTMatchers.h
===
--- clang/include/clang/ASTMatchers/ASTMatchers.h
+++ clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -6452,6 +6452,43 @@
   return false;
 }
 
+/// Matches expressions that match InnerMatcher after any elidable constructor
+/// are stripped off. In C++17 copy elidable constructors are no longer being
+/// generated in the AST as it is not permitted by the standard. They are
+/// however part of the C++14 AST. This means that there are cases where it is
+/// needed to use ``anyOf(cxxConstructorExpr(Inner), Inner)`` to match for both
+/// the C++14 and C++17 AST. Instead of doing this, this matcher can be used as
+/// ``ignoringElidableMoveConstructorCall(Inner)``.
+///
+/// Given
+///
+/// \code
+/// struct H {};
+/// template H B(T A);
+/// void f() {
+///   H D1;
+///   D1 = B(B(1));
+/// }
+/// \endcode
+///
+/// ``cxxOperatorCallExpr(hasArgument(

[PATCH] D63149: Added AST matcher for ignoring elidable constructors

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 204244.
jvikstrom marked an inline comment as done.
jvikstrom added a comment.

- Updated example and fixed edge case in ignoringElidableConstructorCall


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D63149/new/

https://reviews.llvm.org/D63149

Files:
  clang/docs/LibASTMatchersReference.html
  clang/include/clang/ASTMatchers/ASTMatchers.h
  clang/lib/ASTMatchers/Dynamic/Registry.cpp
  clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp

Index: clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
===
--- clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -566,6 +566,81 @@
llvm::make_unique>("x")));
 }
 
+TEST(Matcher, IgnoresElidableConstructors) {
+  auto matcher1 = cxxOperatorCallExpr(hasArgument(
+  1, callExpr(hasArgument(
+ 0, ignoringElidableMoveConstructorCall(callExpr().bind("c"));
+  auto matcher2 = cxxOperatorCallExpr(hasArgument(
+  1, callExpr(hasArgument(0, ignoringElidableMoveConstructorCall(
+ integerLiteral().bind("c"));
+  auto matcher3 =
+  varDecl(hasInitializer(ignoringElidableMoveConstructorCall(callExpr(;
+
+  auto code1 = "struct H {};"
+   "template H B(T A);"
+   "void f(){"
+   "H D1;"
+   "D1=B(B(1));"
+   "}";
+  auto code2 = "struct H {};"
+   "template H B(T A);"
+   "void f(){"
+   "H D1;"
+   "D1=B(1);"
+   "}";
+  auto code3 = "struct H {};"
+   "H G();"
+   "void f(){"
+   "H D = G();"
+   "}";
+
+  EXPECT_TRUE(matchesConditionally(code1, matcher1, true, "-std=c++14"));
+  EXPECT_TRUE(matchesConditionally(code1, matcher1, true, "-std=c++17"));
+
+  EXPECT_TRUE(matchesConditionally(code2, matcher2, true, "-std=c++14"));
+  EXPECT_TRUE(matchesConditionally(code2, matcher2, true, "-std=c++17"));
+
+  EXPECT_TRUE(matchesConditionally(code3, matcher3, true, "-std=c++14"));
+  EXPECT_TRUE(matchesConditionally(code3, matcher3, true, "-std=c++17"));
+}
+
+TEST(Matcher, IgnoresElidableInReturn) {
+  auto matcher = expr(ignoringElidableMoveConstructorCall(declRefExpr()));
+  auto code1 = "struct H{};"
+   "H f(){"
+   "H g;"
+   "return g;"
+   "}";
+  auto code2 = "struct H{};"
+   "H f(){"
+   "return H();"
+   "}";
+  EXPECT_TRUE(matchesConditionally(code1, matcher, true, "-std=c++14"));
+  EXPECT_TRUE(matchesConditionally(code1, matcher, true, "-std=c++17"));
+  EXPECT_TRUE(matchesConditionally(code2, matcher, false, "-std=c++14"));
+  EXPECT_TRUE(matchesConditionally(code2, matcher, false, "-std=c++17"));
+}
+
+TEST(Matcher, IgnoreElidableConstructorDoesNotMatchConstructors) {
+  auto matcher = varDecl(
+  hasInitializer(ignoringElidableMoveConstructorCall(cxxConstructExpr(;
+  auto code = "struct H {};"
+  "void f(){"
+  "H D;"
+  "}";
+  EXPECT_TRUE(matchesConditionally(code, matcher, true, "-std=c++14"));
+  EXPECT_TRUE(matchesConditionally(code, matcher, true, "-std=c++17"));
+};
+
+TEST(Matcher, IgnoresElidableDoesNotPreventMatches) {
+  auto matcher = expr(ignoringElidableMoveConstructorCall(integerLiteral()));
+  auto code = "void f(){"
+  "int D = 10;"
+  "}";
+  EXPECT_TRUE(matchesConditionally(code, matcher, true, "-std=c++14"));
+  EXPECT_TRUE(matchesConditionally(code, matcher, true, "-std=c++17"));
+}
+
 TEST(Matcher, BindTheSameNameInAlternatives) {
   StatementMatcher matcher = anyOf(
 binaryOperator(hasOperatorName("+"),
Index: clang/lib/ASTMatchers/Dynamic/Registry.cpp
===
--- clang/lib/ASTMatchers/Dynamic/Registry.cpp
+++ clang/lib/ASTMatchers/Dynamic/Registry.cpp
@@ -320,6 +320,7 @@
   REGISTER_MATCHER(hasUnqualifiedDesugaredType);
   REGISTER_MATCHER(hasValueType);
   REGISTER_MATCHER(ifStmt);
+  REGISTER_MATCHER(ignoringElidableMoveConstructorCall);
   REGISTER_MATCHER(ignoringImpCasts);
   REGISTER_MATCHER(ignoringImplicit);
   REGISTER_MATCHER(ignoringParenCasts);
Index: clang/include/clang/ASTMatchers/ASTMatchers.h
===
--- clang/include/clang/ASTMatchers/ASTMatchers.h
+++ clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -6452,6 +6452,56 @@
   return false;
 }
 
+/// Matches expressions that match InnerMatcher that are possibly wrapped in an
+/// elidable constructor.
+///
+/// In C++17 copy elidable constructors are no longer being
+/// generated in the AST as it is not permitted by the standard. They are
+/// however part of the AST in C++14 and earlier. Therefore, to write

[PATCH] D63149: Added AST matcher for ignoring elidable constructors

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 204245.
jvikstrom added a comment.

- Using CamelCase and also renamed ignoringElidableMoveConstructorCall to 
ignoringElidableConstructorCall


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D63149/new/

https://reviews.llvm.org/D63149

Files:
  clang/docs/LibASTMatchersReference.html
  clang/include/clang/ASTMatchers/ASTMatchers.h
  clang/lib/ASTMatchers/Dynamic/Registry.cpp
  clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp

Index: clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
===
--- clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -566,6 +566,81 @@
llvm::make_unique>("x")));
 }
 
+TEST(Matcher, IgnoresElidableConstructors) {
+  auto matcher1 = cxxOperatorCallExpr(hasArgument(
+  1, callExpr(hasArgument(
+ 0, ignoringElidableMoveConstructorCall(callExpr().bind("c"));
+  auto matcher2 = cxxOperatorCallExpr(hasArgument(
+  1, callExpr(hasArgument(0, ignoringElidableMoveConstructorCall(
+ integerLiteral().bind("c"));
+  auto matcher3 =
+  varDecl(hasInitializer(ignoringElidableMoveConstructorCall(callExpr(;
+
+  auto code1 = "struct H {};"
+   "template H B(T A);"
+   "void f(){"
+   "H D1;"
+   "D1=B(B(1));"
+   "}";
+  auto code2 = "struct H {};"
+   "template H B(T A);"
+   "void f(){"
+   "H D1;"
+   "D1=B(1);"
+   "}";
+  auto code3 = "struct H {};"
+   "H G();"
+   "void f(){"
+   "H D = G();"
+   "}";
+
+  EXPECT_TRUE(matchesConditionally(code1, matcher1, true, "-std=c++14"));
+  EXPECT_TRUE(matchesConditionally(code1, matcher1, true, "-std=c++17"));
+
+  EXPECT_TRUE(matchesConditionally(code2, matcher2, true, "-std=c++14"));
+  EXPECT_TRUE(matchesConditionally(code2, matcher2, true, "-std=c++17"));
+
+  EXPECT_TRUE(matchesConditionally(code3, matcher3, true, "-std=c++14"));
+  EXPECT_TRUE(matchesConditionally(code3, matcher3, true, "-std=c++17"));
+}
+
+TEST(Matcher, IgnoresElidableInReturn) {
+  auto matcher = expr(ignoringElidableMoveConstructorCall(declRefExpr()));
+  auto code1 = "struct H{};"
+   "H f(){"
+   "H g;"
+   "return g;"
+   "}";
+  auto code2 = "struct H{};"
+   "H f(){"
+   "return H();"
+   "}";
+  EXPECT_TRUE(matchesConditionally(code1, matcher, true, "-std=c++14"));
+  EXPECT_TRUE(matchesConditionally(code1, matcher, true, "-std=c++17"));
+  EXPECT_TRUE(matchesConditionally(code2, matcher, false, "-std=c++14"));
+  EXPECT_TRUE(matchesConditionally(code2, matcher, false, "-std=c++17"));
+}
+
+TEST(Matcher, IgnoreElidableConstructorDoesNotMatchConstructors) {
+  auto matcher = varDecl(
+  hasInitializer(ignoringElidableMoveConstructorCall(cxxConstructExpr(;
+  auto code = "struct H {};"
+  "void f(){"
+  "H D;"
+  "}";
+  EXPECT_TRUE(matchesConditionally(code, matcher, true, "-std=c++14"));
+  EXPECT_TRUE(matchesConditionally(code, matcher, true, "-std=c++17"));
+};
+
+TEST(Matcher, IgnoresElidableDoesNotPreventMatches) {
+  auto matcher = expr(ignoringElidableMoveConstructorCall(integerLiteral()));
+  auto code = "void f(){"
+  "int D = 10;"
+  "}";
+  EXPECT_TRUE(matchesConditionally(code, matcher, true, "-std=c++14"));
+  EXPECT_TRUE(matchesConditionally(code, matcher, true, "-std=c++17"));
+}
+
 TEST(Matcher, BindTheSameNameInAlternatives) {
   StatementMatcher matcher = anyOf(
 binaryOperator(hasOperatorName("+"),
Index: clang/lib/ASTMatchers/Dynamic/Registry.cpp
===
--- clang/lib/ASTMatchers/Dynamic/Registry.cpp
+++ clang/lib/ASTMatchers/Dynamic/Registry.cpp
@@ -320,6 +320,7 @@
   REGISTER_MATCHER(hasUnqualifiedDesugaredType);
   REGISTER_MATCHER(hasValueType);
   REGISTER_MATCHER(ifStmt);
+  REGISTER_MATCHER(ignoringElidableMoveConstructorCall);
   REGISTER_MATCHER(ignoringImpCasts);
   REGISTER_MATCHER(ignoringImplicit);
   REGISTER_MATCHER(ignoringParenCasts);
Index: clang/include/clang/ASTMatchers/ASTMatchers.h
===
--- clang/include/clang/ASTMatchers/ASTMatchers.h
+++ clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -6452,6 +6452,56 @@
   return false;
 }
 
+/// Matches expressions that match InnerMatcher that are possibly wrapped in an
+/// elidable constructor.
+///
+/// In C++17 copy elidable constructors are no longer being
+/// generated in the AST as it is not permitted by the standard. They are
+/// however part of the AST in C++14 and earlier. Therefore, to write a matcher

[PATCH] D63149: Added AST matcher for ignoring elidable constructors

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 204269.
jvikstrom marked 3 inline comments as done.
jvikstrom added a comment.

- Added match conditionally overload to control in what language standard a 
match should run in


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D63149/new/

https://reviews.llvm.org/D63149

Files:
  clang/docs/LibASTMatchersReference.html
  clang/include/clang/ASTMatchers/ASTMatchers.h
  clang/lib/ASTMatchers/Dynamic/Registry.cpp
  clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
  clang/unittests/ASTMatchers/ASTMatchersTest.h

Index: clang/unittests/ASTMatchers/ASTMatchersTest.h
===
--- clang/unittests/ASTMatchers/ASTMatchersTest.h
+++ clang/unittests/ASTMatchers/ASTMatchersTest.h
@@ -57,6 +57,17 @@
   const std::unique_ptr FindResultReviewer;
 };
 
+enum class LanguageMode : int {
+  CXX11 = 0,
+  CXX14 = 1,
+  CXX17 = 2,
+  CXX20 = 3,
+  CXX11OrLater,
+  CXX14OrLater,
+  CXX17OrLater,
+  CXX20OrLater
+};
+
 template 
 testing::AssertionResult matchesConditionally(
 const std::string &Code, const T &AMatcher, bool ExpectMatch,
@@ -116,14 +127,56 @@
 }
 
 template 
-testing::AssertionResult matches(const std::string &Code, const T &AMatcher) {
-  return matchesConditionally(Code, AMatcher, true, "-std=c++11");
+testing::AssertionResult
+matchesConditionally(const std::string &Code, const T &AMatcher,
+ bool ExpectMatch, const LanguageMode &Mode) {
+  std::vector LangModeStrings{"-std=c++11", "-std=c++14",
+   "-std=c++17", "-std=c++20"};
+  std::vector LangModes;
+  switch (Mode) {
+  case LanguageMode::CXX11:
+  case LanguageMode::CXX14:
+  case LanguageMode::CXX17:
+  case LanguageMode::CXX20:
+LangModes = {Mode};
+break;
+  case LanguageMode::CXX11OrLater:
+LangModes = {LanguageMode::CXX11, LanguageMode::CXX14, LanguageMode::CXX17, LanguageMode::CXX20};
+break;
+  case LanguageMode::CXX14OrLater:
+LangModes = {LanguageMode::CXX14, LanguageMode::CXX17, LanguageMode::CXX20};
+break;
+  case LanguageMode::CXX17OrLater:
+LangModes = {LanguageMode::CXX17, LanguageMode::CXX20};
+break;
+  case LanguageMode::CXX20OrLater:
+LangModes = {LanguageMode::CXX20};
+  };
+
+  for (auto Mode : LangModes) {
+auto LangModeArg = LangModeStrings[static_cast(Mode)];
+auto Result =
+matchesConditionally(Code, AMatcher, ExpectMatch, LangModeArg);
+if (!Result) {
+  return Result;
+}
+  }
+
+  return testing::AssertionSuccess();
 }
 
 template 
-testing::AssertionResult notMatches(const std::string &Code,
-const T &AMatcher) {
-  return matchesConditionally(Code, AMatcher, false, "-std=c++11");
+testing::AssertionResult
+matches(const std::string &Code, const T &AMatcher,
+const LanguageMode &Modes = LanguageMode::CXX11) {
+  return matchesConditionally(Code, AMatcher, true, Modes);
+}
+
+template 
+testing::AssertionResult
+notMatches(const std::string &Code, const T &AMatcher,
+   const LanguageMode &Modes = LanguageMode::CXX11) {
+  return matchesConditionally(Code, AMatcher, false, Modes);
 }
 
 template 
Index: clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
===
--- clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -566,6 +566,76 @@
llvm::make_unique>("x")));
 }
 
+TEST(Matcher, IgnoresElidableConstructors) {
+  auto matcher1 = cxxOperatorCallExpr(hasArgument(
+  1, callExpr(hasArgument(
+ 0, ignoringElidableConstructorCall(callExpr());
+  auto matcher2 = cxxOperatorCallExpr(hasArgument(
+  1, callExpr(hasArgument(0, ignoringElidableConstructorCall(
+ integerLiteral());
+  auto matcher3 =
+  varDecl(hasInitializer(
+anyOf(
+ignoringElidableConstructorCall(callExpr()),
+exprWithCleanups(has(ignoringElidableConstructorCall(callExpr(
+)));
+
+  auto code1 = "struct H {};"
+   "template H B(T A);"
+   "void f() {"
+   "  H D1;"
+   "  D1 = B(B(1));"
+   "}";
+  auto code2 = "struct H {};"
+   "template H B(T A);"
+   "void f() {"
+   "  H D1;"
+   "  D1 = B(1);"
+   "}";
+  auto code3 = "struct H {};"
+   "H G();"
+   "void f() {"
+   "  H D = G();"
+   "}";
+
+  EXPECT_TRUE(matches(code1, matcher1, LanguageMode::CXX11OrLater));
+  EXPECT_TRUE(matches(code2, matcher2, LanguageMode::CXX11OrLater));
+  EXPECT_TRUE(matches(code3, matcher3, LanguageMode::CXX11OrLater));
+}
+
+TEST(Matcher, IgnoresElidableInReturn) {
+  auto matcher = expr(ignoringElidableConstructorCall(

[PATCH] D63149: Added AST matcher for ignoring elidable constructors

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 204277.
jvikstrom marked 10 inline comments as done.
jvikstrom added a comment.

- Fixed wrong formatting in test code


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D63149/new/

https://reviews.llvm.org/D63149

Files:
  clang/docs/LibASTMatchersReference.html
  clang/include/clang/ASTMatchers/ASTMatchers.h
  clang/lib/ASTMatchers/Dynamic/Registry.cpp
  clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
  clang/unittests/ASTMatchers/ASTMatchersTest.h

Index: clang/unittests/ASTMatchers/ASTMatchersTest.h
===
--- clang/unittests/ASTMatchers/ASTMatchersTest.h
+++ clang/unittests/ASTMatchers/ASTMatchersTest.h
@@ -57,6 +57,17 @@
   const std::unique_ptr FindResultReviewer;
 };
 
+enum class LanguageMode : int {
+  CXX11 = 0,
+  CXX14 = 1,
+  CXX17 = 2,
+  CXX20 = 3,
+  CXX11OrLater,
+  CXX14OrLater,
+  CXX17OrLater,
+  CXX20OrLater
+};
+
 template 
 testing::AssertionResult matchesConditionally(
 const std::string &Code, const T &AMatcher, bool ExpectMatch,
@@ -116,14 +127,56 @@
 }
 
 template 
-testing::AssertionResult matches(const std::string &Code, const T &AMatcher) {
-  return matchesConditionally(Code, AMatcher, true, "-std=c++11");
+testing::AssertionResult
+matchesConditionally(const std::string &Code, const T &AMatcher,
+ bool ExpectMatch, const LanguageMode &Mode) {
+  std::vector LangModeStrings{"-std=c++11", "-std=c++14",
+   "-std=c++17", "-std=c++20"};
+  std::vector LangModes;
+  switch (Mode) {
+  case LanguageMode::CXX11:
+  case LanguageMode::CXX14:
+  case LanguageMode::CXX17:
+  case LanguageMode::CXX20:
+LangModes = {Mode};
+break;
+  case LanguageMode::CXX11OrLater:
+LangModes = {LanguageMode::CXX11, LanguageMode::CXX14, LanguageMode::CXX17, LanguageMode::CXX20};
+break;
+  case LanguageMode::CXX14OrLater:
+LangModes = {LanguageMode::CXX14, LanguageMode::CXX17, LanguageMode::CXX20};
+break;
+  case LanguageMode::CXX17OrLater:
+LangModes = {LanguageMode::CXX17, LanguageMode::CXX20};
+break;
+  case LanguageMode::CXX20OrLater:
+LangModes = {LanguageMode::CXX20};
+  };
+
+  for (auto Mode : LangModes) {
+auto LangModeArg = LangModeStrings[static_cast(Mode)];
+auto Result =
+matchesConditionally(Code, AMatcher, ExpectMatch, LangModeArg);
+if (!Result) {
+  return Result;
+}
+  }
+
+  return testing::AssertionSuccess();
 }
 
 template 
-testing::AssertionResult notMatches(const std::string &Code,
-const T &AMatcher) {
-  return matchesConditionally(Code, AMatcher, false, "-std=c++11");
+testing::AssertionResult
+matches(const std::string &Code, const T &AMatcher,
+const LanguageMode &Modes = LanguageMode::CXX11) {
+  return matchesConditionally(Code, AMatcher, true, Modes);
+}
+
+template 
+testing::AssertionResult
+notMatches(const std::string &Code, const T &AMatcher,
+   const LanguageMode &Modes = LanguageMode::CXX11) {
+  return matchesConditionally(Code, AMatcher, false, Modes);
 }
 
 template 
Index: clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
===
--- clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -566,6 +566,73 @@
llvm::make_unique>("x")));
 }
 
+TEST(Matcher, IgnoresElidableConstructors) {
+  auto matcher1 = cxxOperatorCallExpr(hasArgument(
+  1,
+  callExpr(hasArgument(0, ignoringElidableConstructorCall(callExpr());
+  auto matcher2 = cxxOperatorCallExpr(
+  hasArgument(1, callExpr(hasArgument(0, ignoringElidableConstructorCall(
+ integerLiteral());
+  auto matcher3 = varDecl(hasInitializer(anyOf(
+  ignoringElidableConstructorCall(callExpr()),
+  exprWithCleanups(has(ignoringElidableConstructorCall(callExpr()));
+
+  auto code1 = "struct H {};"
+   "template H B(T A);"
+   "void f() {"
+   "  H D1;"
+   "  D1 = B(B(1));"
+   "}";
+  auto code2 = "struct H {};"
+   "template H B(T A);"
+   "void f() {"
+   "  H D1;"
+   "  D1 = B(1);"
+   "}";
+  auto code3 = "struct H {};"
+   "H G();"
+   "void f() {"
+   "  H D = G();"
+   "}";
+
+  EXPECT_TRUE(matches(code1, matcher1, LanguageMode::CXX11OrLater));
+  EXPECT_TRUE(matches(code2, matcher2, LanguageMode::CXX11OrLater));
+  EXPECT_TRUE(matches(code3, matcher3, LanguageMode::CXX11OrLater));
+}
+
+TEST(Matcher, IgnoresElidableInReturn) {
+  auto matcher = expr(ignoringElidableConstructorCall(declRefExpr()));
+  EXPECT_TRUE(matches("struct H {};"
+  "H f() 

[PATCH] D63149: Added AST matcher for ignoring elidable constructors

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 204282.
jvikstrom marked 7 inline comments as done.
jvikstrom added a comment.

- Using switch for choosing language standard


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D63149/new/

https://reviews.llvm.org/D63149

Files:
  clang/docs/LibASTMatchersReference.html
  clang/include/clang/ASTMatchers/ASTMatchers.h
  clang/lib/ASTMatchers/Dynamic/Registry.cpp
  clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
  clang/unittests/ASTMatchers/ASTMatchersTest.h

Index: clang/unittests/ASTMatchers/ASTMatchersTest.h
===
--- clang/unittests/ASTMatchers/ASTMatchersTest.h
+++ clang/unittests/ASTMatchers/ASTMatchersTest.h
@@ -13,6 +13,7 @@
 #include "clang/Frontend/ASTUnit.h"
 #include "clang/Tooling/Tooling.h"
 #include "gtest/gtest.h"
+#include 
 
 namespace clang {
 namespace ast_matchers {
@@ -57,6 +58,17 @@
   const std::unique_ptr FindResultReviewer;
 };
 
+enum class LanguageMode {
+  CXX11,
+  CXX14,
+  CXX17,
+  CXX2A,
+  CXX11OrLater,
+  CXX14OrLater,
+  CXX17OrLater,
+  CXX2AOrLater
+};
+
 template 
 testing::AssertionResult matchesConditionally(
 const std::string &Code, const T &AMatcher, bool ExpectMatch,
@@ -116,14 +128,73 @@
 }
 
 template 
-testing::AssertionResult matches(const std::string &Code, const T &AMatcher) {
-  return matchesConditionally(Code, AMatcher, true, "-std=c++11");
+testing::AssertionResult
+matchesConditionally(const std::string &Code, const T &AMatcher,
+ bool ExpectMatch, const LanguageMode &Mode) {
+  std::vector LangModes;
+  switch (Mode) {
+  case LanguageMode::CXX11:
+  case LanguageMode::CXX14:
+  case LanguageMode::CXX17:
+  case LanguageMode::CXX2A:
+LangModes = {Mode};
+break;
+  case LanguageMode::CXX11OrLater:
+LangModes = {LanguageMode::CXX11, LanguageMode::CXX14, LanguageMode::CXX17,
+ LanguageMode::CXX2A};
+break;
+  case LanguageMode::CXX14OrLater:
+LangModes = {LanguageMode::CXX14, LanguageMode::CXX17, LanguageMode::CXX2A};
+break;
+  case LanguageMode::CXX17OrLater:
+LangModes = {LanguageMode::CXX17, LanguageMode::CXX2A};
+break;
+  case LanguageMode::CXX2AOrLater:
+LangModes = {LanguageMode::CXX2A};
+  }
+
+  for (auto Mode : LangModes) {
+std::string LangModeArg;
+switch (Mode) {
+case LanguageMode::CXX11OrLater:
+case LanguageMode::CXX11:
+  LangModeArg = "-std=c++11";
+  break;
+case LanguageMode::CXX14OrLater:
+case LanguageMode::CXX14:
+  LangModeArg = "-std=c++14";
+  break;
+case LanguageMode::CXX17OrLater:
+case LanguageMode::CXX17:
+  LangModeArg = "-std=c++17";
+  break;
+case LanguageMode::CXX2AOrLater:
+case LanguageMode::CXX2A:
+  LangModeArg = "-std=c++2a";
+}
+
+auto Result =
+matchesConditionally(Code, AMatcher, ExpectMatch, LangModeArg);
+if (!Result) {
+  return Result;
+}
+  }
+
+  return testing::AssertionSuccess();
 }
 
 template 
-testing::AssertionResult notMatches(const std::string &Code,
-const T &AMatcher) {
-  return matchesConditionally(Code, AMatcher, false, "-std=c++11");
+testing::AssertionResult
+matches(const std::string &Code, const T &AMatcher,
+const LanguageMode &Modes = LanguageMode::CXX11) {
+  return matchesConditionally(Code, AMatcher, true, Modes);
+}
+
+template 
+testing::AssertionResult
+notMatches(const std::string &Code, const T &AMatcher,
+   const LanguageMode &Modes = LanguageMode::CXX11) {
+  return matchesConditionally(Code, AMatcher, false, Modes);
 }
 
 template 
Index: clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
===
--- clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -566,6 +566,74 @@
llvm::make_unique>("x")));
 }
 
+TEST(Matcher, IgnoresElidableConstructors) {
+  EXPECT_TRUE(
+  matches("struct H {};"
+  "template H B(T A);"
+  "void f() {"
+  "  H D1;"
+  "  D1 = B(B(1));"
+  "}",
+  cxxOperatorCallExpr(hasArgument(
+  1, callExpr(hasArgument(
+ 0, ignoringElidableConstructorCall(callExpr()),
+  LanguageMode::CXX11OrLater));
+  EXPECT_TRUE(
+  matches("struct H {};"
+  "template H B(T A);"
+  "void f() {"
+  "  H D1;"
+  "  D1 = B(1);"
+  "}",
+  cxxOperatorCallExpr(hasArgument(
+  1, callExpr(hasArgument(0, ignoringElidableConstructorCall(
+ integerLiteral()),
+  LanguageMode::CXX11OrLater));
+  EXPECT_TRUE(matches(
+  "struct H {};"
+  "H G();"
+ 

[PATCH] D63149: Added AST matcher for ignoring elidable constructors

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 204437.
jvikstrom marked an inline comment as done.
jvikstrom added a comment.

- Added default to switch for language modes


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D63149/new/

https://reviews.llvm.org/D63149

Files:
  clang/docs/LibASTMatchersReference.html
  clang/include/clang/ASTMatchers/ASTMatchers.h
  clang/lib/ASTMatchers/Dynamic/Registry.cpp
  clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
  clang/unittests/ASTMatchers/ASTMatchersTest.h

Index: clang/unittests/ASTMatchers/ASTMatchersTest.h
===
--- clang/unittests/ASTMatchers/ASTMatchersTest.h
+++ clang/unittests/ASTMatchers/ASTMatchersTest.h
@@ -13,6 +13,7 @@
 #include "clang/Frontend/ASTUnit.h"
 #include "clang/Tooling/Tooling.h"
 #include "gtest/gtest.h"
+#include 
 
 namespace clang {
 namespace ast_matchers {
@@ -57,6 +58,17 @@
   const std::unique_ptr FindResultReviewer;
 };
 
+enum class LanguageMode {
+  Cxx11,
+  Cxx14,
+  Cxx17,
+  Cxx2a,
+  Cxx11OrLater,
+  Cxx14OrLater,
+  Cxx17OrLater,
+  Cxx2aOrLater
+};
+
 template 
 testing::AssertionResult matchesConditionally(
 const std::string &Code, const T &AMatcher, bool ExpectMatch,
@@ -116,14 +128,72 @@
 }
 
 template 
-testing::AssertionResult matches(const std::string &Code, const T &AMatcher) {
-  return matchesConditionally(Code, AMatcher, true, "-std=c++11");
+testing::AssertionResult
+matchesConditionally(const std::string &Code, const T &AMatcher,
+ bool ExpectMatch, const LanguageMode &Mode) {
+  std::vector LangModes;
+  switch (Mode) {
+  case LanguageMode::Cxx11:
+  case LanguageMode::Cxx14:
+  case LanguageMode::Cxx17:
+  case LanguageMode::Cxx2a:
+LangModes = {Mode};
+break;
+  case LanguageMode::Cxx11OrLater:
+LangModes = {LanguageMode::Cxx11, LanguageMode::Cxx14, LanguageMode::Cxx17,
+ LanguageMode::Cxx2a};
+break;
+  case LanguageMode::Cxx14OrLater:
+LangModes = {LanguageMode::Cxx14, LanguageMode::Cxx17, LanguageMode::Cxx2a};
+break;
+  case LanguageMode::Cxx17OrLater:
+LangModes = {LanguageMode::Cxx17, LanguageMode::Cxx2a};
+break;
+  case LanguageMode::Cxx2aOrLater:
+LangModes = {LanguageMode::Cxx2a};
+  }
+
+  for (auto Mode : LangModes) {
+std::string LangModeArg;
+switch (Mode) {
+case LanguageMode::Cxx11:
+  LangModeArg = "-std=c++11";
+  break;
+case LanguageMode::Cxx14:
+  LangModeArg = "-std=c++14";
+  break;
+case LanguageMode::Cxx17:
+  LangModeArg = "-std=c++17";
+  break;
+case LanguageMode::Cxx2a:
+  LangModeArg = "-std=c++2a";
+  break;
+default:
+  llvm_unreachable("Invalid language mode");
+}
+
+auto Result =
+matchesConditionally(Code, AMatcher, ExpectMatch, LangModeArg);
+if (!Result) {
+  return Result;
+}
+  }
+
+  return testing::AssertionSuccess();
 }
 
 template 
-testing::AssertionResult notMatches(const std::string &Code,
-const T &AMatcher) {
-  return matchesConditionally(Code, AMatcher, false, "-std=c++11");
+testing::AssertionResult
+matches(const std::string &Code, const T &AMatcher,
+const LanguageMode &Modes = LanguageMode::Cxx11) {
+  return matchesConditionally(Code, AMatcher, true, Modes);
+}
+
+template 
+testing::AssertionResult
+notMatches(const std::string &Code, const T &AMatcher,
+   const LanguageMode &Modes = LanguageMode::Cxx11) {
+  return matchesConditionally(Code, AMatcher, false, Modes);
 }
 
 template 
Index: clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
===
--- clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -566,6 +566,74 @@
llvm::make_unique>("x")));
 }
 
+TEST(Matcher, IgnoresElidableConstructors) {
+  EXPECT_TRUE(
+  matches("struct H {};"
+  "template H B(T A);"
+  "void f() {"
+  "  H D1;"
+  "  D1 = B(B(1));"
+  "}",
+  cxxOperatorCallExpr(hasArgument(
+  1, callExpr(hasArgument(
+ 0, ignoringElidableConstructorCall(callExpr()),
+  LanguageMode::Cxx11OrLater));
+  EXPECT_TRUE(
+  matches("struct H {};"
+  "template H B(T A);"
+  "void f() {"
+  "  H D1;"
+  "  D1 = B(1);"
+  "}",
+  cxxOperatorCallExpr(hasArgument(
+  1, callExpr(hasArgument(0, ignoringElidableConstructorCall(
+ integerLiteral()),
+  LanguageMode::Cxx11OrLater));
+  EXPECT_TRUE(matches(
+  "struct H {};"
+  "H G();"
+  "void f() {"
+  "  H D = G();"
+  "}",
+  varDecl(hasInitializ

[PATCH] D63149: Added AST matcher for ignoring elidable constructors

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 204442.
jvikstrom added a comment.

- Updated outdated docs


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D63149/new/

https://reviews.llvm.org/D63149

Files:
  clang/docs/LibASTMatchersReference.html
  clang/include/clang/ASTMatchers/ASTMatchers.h
  clang/lib/ASTMatchers/Dynamic/Registry.cpp
  clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
  clang/unittests/ASTMatchers/ASTMatchersTest.h

Index: clang/unittests/ASTMatchers/ASTMatchersTest.h
===
--- clang/unittests/ASTMatchers/ASTMatchersTest.h
+++ clang/unittests/ASTMatchers/ASTMatchersTest.h
@@ -13,6 +13,7 @@
 #include "clang/Frontend/ASTUnit.h"
 #include "clang/Tooling/Tooling.h"
 #include "gtest/gtest.h"
+#include 
 
 namespace clang {
 namespace ast_matchers {
@@ -57,6 +58,17 @@
   const std::unique_ptr FindResultReviewer;
 };
 
+enum class LanguageMode {
+  Cxx11,
+  Cxx14,
+  Cxx17,
+  Cxx2a,
+  Cxx11OrLater,
+  Cxx14OrLater,
+  Cxx17OrLater,
+  Cxx2aOrLater
+};
+
 template 
 testing::AssertionResult matchesConditionally(
 const std::string &Code, const T &AMatcher, bool ExpectMatch,
@@ -116,14 +128,72 @@
 }
 
 template 
-testing::AssertionResult matches(const std::string &Code, const T &AMatcher) {
-  return matchesConditionally(Code, AMatcher, true, "-std=c++11");
+testing::AssertionResult
+matchesConditionally(const std::string &Code, const T &AMatcher,
+ bool ExpectMatch, const LanguageMode &Mode) {
+  std::vector LangModes;
+  switch (Mode) {
+  case LanguageMode::Cxx11:
+  case LanguageMode::Cxx14:
+  case LanguageMode::Cxx17:
+  case LanguageMode::Cxx2a:
+LangModes = {Mode};
+break;
+  case LanguageMode::Cxx11OrLater:
+LangModes = {LanguageMode::Cxx11, LanguageMode::Cxx14, LanguageMode::Cxx17,
+ LanguageMode::Cxx2a};
+break;
+  case LanguageMode::Cxx14OrLater:
+LangModes = {LanguageMode::Cxx14, LanguageMode::Cxx17, LanguageMode::Cxx2a};
+break;
+  case LanguageMode::Cxx17OrLater:
+LangModes = {LanguageMode::Cxx17, LanguageMode::Cxx2a};
+break;
+  case LanguageMode::Cxx2aOrLater:
+LangModes = {LanguageMode::Cxx2a};
+  }
+
+  for (auto Mode : LangModes) {
+std::string LangModeArg;
+switch (Mode) {
+case LanguageMode::Cxx11:
+  LangModeArg = "-std=c++11";
+  break;
+case LanguageMode::Cxx14:
+  LangModeArg = "-std=c++14";
+  break;
+case LanguageMode::Cxx17:
+  LangModeArg = "-std=c++17";
+  break;
+case LanguageMode::Cxx2a:
+  LangModeArg = "-std=c++2a";
+  break;
+default:
+  llvm_unreachable("Invalid language mode");
+}
+
+auto Result =
+matchesConditionally(Code, AMatcher, ExpectMatch, LangModeArg);
+if (!Result) {
+  return Result;
+}
+  }
+
+  return testing::AssertionSuccess();
 }
 
 template 
-testing::AssertionResult notMatches(const std::string &Code,
-const T &AMatcher) {
-  return matchesConditionally(Code, AMatcher, false, "-std=c++11");
+testing::AssertionResult
+matches(const std::string &Code, const T &AMatcher,
+const LanguageMode &Modes = LanguageMode::Cxx11) {
+  return matchesConditionally(Code, AMatcher, true, Modes);
+}
+
+template 
+testing::AssertionResult
+notMatches(const std::string &Code, const T &AMatcher,
+   const LanguageMode &Modes = LanguageMode::Cxx11) {
+  return matchesConditionally(Code, AMatcher, false, Modes);
 }
 
 template 
Index: clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
===
--- clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -566,6 +566,74 @@
llvm::make_unique>("x")));
 }
 
+TEST(Matcher, IgnoresElidableConstructors) {
+  EXPECT_TRUE(
+  matches("struct H {};"
+  "template H B(T A);"
+  "void f() {"
+  "  H D1;"
+  "  D1 = B(B(1));"
+  "}",
+  cxxOperatorCallExpr(hasArgument(
+  1, callExpr(hasArgument(
+ 0, ignoringElidableConstructorCall(callExpr()),
+  LanguageMode::Cxx11OrLater));
+  EXPECT_TRUE(
+  matches("struct H {};"
+  "template H B(T A);"
+  "void f() {"
+  "  H D1;"
+  "  D1 = B(1);"
+  "}",
+  cxxOperatorCallExpr(hasArgument(
+  1, callExpr(hasArgument(0, ignoringElidableConstructorCall(
+ integerLiteral()),
+  LanguageMode::Cxx11OrLater));
+  EXPECT_TRUE(matches(
+  "struct H {};"
+  "H G();"
+  "void f() {"
+  "  H D = G();"
+  "}",
+  varDecl(hasInitializer(anyOf(
+  ignoringElidableConstructorCall(callExpr()),

[PATCH] D63149: Added AST matcher for ignoring elidable constructors

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 204443.
jvikstrom marked an inline comment as done.
jvikstrom added a comment.

- Removed dependency on unordered map


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D63149/new/

https://reviews.llvm.org/D63149

Files:
  clang/docs/LibASTMatchersReference.html
  clang/include/clang/ASTMatchers/ASTMatchers.h
  clang/lib/ASTMatchers/Dynamic/Registry.cpp
  clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
  clang/unittests/ASTMatchers/ASTMatchersTest.h

Index: clang/unittests/ASTMatchers/ASTMatchersTest.h
===
--- clang/unittests/ASTMatchers/ASTMatchersTest.h
+++ clang/unittests/ASTMatchers/ASTMatchersTest.h
@@ -57,6 +57,17 @@
   const std::unique_ptr FindResultReviewer;
 };
 
+enum class LanguageMode {
+  Cxx11,
+  Cxx14,
+  Cxx17,
+  Cxx2a,
+  Cxx11OrLater,
+  Cxx14OrLater,
+  Cxx17OrLater,
+  Cxx2aOrLater
+};
+
 template 
 testing::AssertionResult matchesConditionally(
 const std::string &Code, const T &AMatcher, bool ExpectMatch,
@@ -116,14 +127,72 @@
 }
 
 template 
-testing::AssertionResult matches(const std::string &Code, const T &AMatcher) {
-  return matchesConditionally(Code, AMatcher, true, "-std=c++11");
+testing::AssertionResult
+matchesConditionally(const std::string &Code, const T &AMatcher,
+ bool ExpectMatch, const LanguageMode &Mode) {
+  std::vector LangModes;
+  switch (Mode) {
+  case LanguageMode::Cxx11:
+  case LanguageMode::Cxx14:
+  case LanguageMode::Cxx17:
+  case LanguageMode::Cxx2a:
+LangModes = {Mode};
+break;
+  case LanguageMode::Cxx11OrLater:
+LangModes = {LanguageMode::Cxx11, LanguageMode::Cxx14, LanguageMode::Cxx17,
+ LanguageMode::Cxx2a};
+break;
+  case LanguageMode::Cxx14OrLater:
+LangModes = {LanguageMode::Cxx14, LanguageMode::Cxx17, LanguageMode::Cxx2a};
+break;
+  case LanguageMode::Cxx17OrLater:
+LangModes = {LanguageMode::Cxx17, LanguageMode::Cxx2a};
+break;
+  case LanguageMode::Cxx2aOrLater:
+LangModes = {LanguageMode::Cxx2a};
+  }
+
+  for (auto Mode : LangModes) {
+std::string LangModeArg;
+switch (Mode) {
+case LanguageMode::Cxx11:
+  LangModeArg = "-std=c++11";
+  break;
+case LanguageMode::Cxx14:
+  LangModeArg = "-std=c++14";
+  break;
+case LanguageMode::Cxx17:
+  LangModeArg = "-std=c++17";
+  break;
+case LanguageMode::Cxx2a:
+  LangModeArg = "-std=c++2a";
+  break;
+default:
+  llvm_unreachable("Invalid language mode");
+}
+
+auto Result =
+matchesConditionally(Code, AMatcher, ExpectMatch, LangModeArg);
+if (!Result) {
+  return Result;
+}
+  }
+
+  return testing::AssertionSuccess();
 }
 
 template 
-testing::AssertionResult notMatches(const std::string &Code,
-const T &AMatcher) {
-  return matchesConditionally(Code, AMatcher, false, "-std=c++11");
+testing::AssertionResult
+matches(const std::string &Code, const T &AMatcher,
+const LanguageMode &Modes = LanguageMode::Cxx11) {
+  return matchesConditionally(Code, AMatcher, true, Modes);
+}
+
+template 
+testing::AssertionResult
+notMatches(const std::string &Code, const T &AMatcher,
+   const LanguageMode &Modes = LanguageMode::Cxx11) {
+  return matchesConditionally(Code, AMatcher, false, Modes);
 }
 
 template 
Index: clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
===
--- clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -566,6 +566,74 @@
llvm::make_unique>("x")));
 }
 
+TEST(Matcher, IgnoresElidableConstructors) {
+  EXPECT_TRUE(
+  matches("struct H {};"
+  "template H B(T A);"
+  "void f() {"
+  "  H D1;"
+  "  D1 = B(B(1));"
+  "}",
+  cxxOperatorCallExpr(hasArgument(
+  1, callExpr(hasArgument(
+ 0, ignoringElidableConstructorCall(callExpr()),
+  LanguageMode::Cxx11OrLater));
+  EXPECT_TRUE(
+  matches("struct H {};"
+  "template H B(T A);"
+  "void f() {"
+  "  H D1;"
+  "  D1 = B(1);"
+  "}",
+  cxxOperatorCallExpr(hasArgument(
+  1, callExpr(hasArgument(0, ignoringElidableConstructorCall(
+ integerLiteral()),
+  LanguageMode::Cxx11OrLater));
+  EXPECT_TRUE(matches(
+  "struct H {};"
+  "H G();"
+  "void f() {"
+  "  H D = G();"
+  "}",
+  varDecl(hasInitializer(anyOf(
+  ignoringElidableConstructorCall(callExpr()),
+  exprWithCleanups(has(ignoringElidableConstructorCall(callExpr())),
+  LanguageMode::Cxx11OrLater)

[PATCH] D63149: Added AST matcher for ignoring elidable constructors

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 204447.
jvikstrom marked 3 inline comments as done.
jvikstrom added a comment.

- Removed unnecessary return


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D63149/new/

https://reviews.llvm.org/D63149

Files:
  clang/docs/LibASTMatchersReference.html
  clang/include/clang/ASTMatchers/ASTMatchers.h
  clang/lib/ASTMatchers/Dynamic/Registry.cpp
  clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
  clang/unittests/ASTMatchers/ASTMatchersTest.h

Index: clang/unittests/ASTMatchers/ASTMatchersTest.h
===
--- clang/unittests/ASTMatchers/ASTMatchersTest.h
+++ clang/unittests/ASTMatchers/ASTMatchersTest.h
@@ -57,6 +57,17 @@
   const std::unique_ptr FindResultReviewer;
 };
 
+enum class LanguageMode {
+  Cxx11,
+  Cxx14,
+  Cxx17,
+  Cxx2a,
+  Cxx11OrLater,
+  Cxx14OrLater,
+  Cxx17OrLater,
+  Cxx2aOrLater
+};
+
 template 
 testing::AssertionResult matchesConditionally(
 const std::string &Code, const T &AMatcher, bool ExpectMatch,
@@ -116,14 +127,71 @@
 }
 
 template 
-testing::AssertionResult matches(const std::string &Code, const T &AMatcher) {
-  return matchesConditionally(Code, AMatcher, true, "-std=c++11");
+testing::AssertionResult
+matchesConditionally(const std::string &Code, const T &AMatcher,
+ bool ExpectMatch, const LanguageMode &Mode) {
+  std::vector LangModes;
+  switch (Mode) {
+  case LanguageMode::Cxx11:
+  case LanguageMode::Cxx14:
+  case LanguageMode::Cxx17:
+  case LanguageMode::Cxx2a:
+LangModes = {Mode};
+break;
+  case LanguageMode::Cxx11OrLater:
+LangModes = {LanguageMode::Cxx11, LanguageMode::Cxx14, LanguageMode::Cxx17,
+ LanguageMode::Cxx2a};
+break;
+  case LanguageMode::Cxx14OrLater:
+LangModes = {LanguageMode::Cxx14, LanguageMode::Cxx17, LanguageMode::Cxx2a};
+break;
+  case LanguageMode::Cxx17OrLater:
+LangModes = {LanguageMode::Cxx17, LanguageMode::Cxx2a};
+break;
+  case LanguageMode::Cxx2aOrLater:
+LangModes = {LanguageMode::Cxx2a};
+  }
+
+  for (auto Mode : LangModes) {
+std::string LangModeArg;
+switch (Mode) {
+case LanguageMode::Cxx11:
+  LangModeArg = "-std=c++11";
+  break;
+case LanguageMode::Cxx14:
+  LangModeArg = "-std=c++14";
+  break;
+case LanguageMode::Cxx17:
+  LangModeArg = "-std=c++17";
+  break;
+case LanguageMode::Cxx2a:
+  LangModeArg = "-std=c++2a";
+  break;
+default:
+  llvm_unreachable("Invalid language mode");
+}
+
+auto Result =
+matchesConditionally(Code, AMatcher, ExpectMatch, LangModeArg);
+if (!Result)
+  return Result;
+  }
+
+  return testing::AssertionSuccess();
+}
+
+template 
+testing::AssertionResult
+matches(const std::string &Code, const T &AMatcher,
+const LanguageMode &Modes = LanguageMode::Cxx11) {
+  return matchesConditionally(Code, AMatcher, true, Modes);
 }
 
 template 
-testing::AssertionResult notMatches(const std::string &Code,
-const T &AMatcher) {
-  return matchesConditionally(Code, AMatcher, false, "-std=c++11");
+testing::AssertionResult
+notMatches(const std::string &Code, const T &AMatcher,
+   const LanguageMode &Modes = LanguageMode::Cxx11) {
+  return matchesConditionally(Code, AMatcher, false, Modes);
 }
 
 template 
Index: clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
===
--- clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -566,6 +566,74 @@
llvm::make_unique>("x")));
 }
 
+TEST(Matcher, IgnoresElidableConstructors) {
+  EXPECT_TRUE(
+  matches("struct H {};"
+  "template H B(T A);"
+  "void f() {"
+  "  H D1;"
+  "  D1 = B(B(1));"
+  "}",
+  cxxOperatorCallExpr(hasArgument(
+  1, callExpr(hasArgument(
+ 0, ignoringElidableConstructorCall(callExpr()),
+  LanguageMode::Cxx11OrLater));
+  EXPECT_TRUE(
+  matches("struct H {};"
+  "template H B(T A);"
+  "void f() {"
+  "  H D1;"
+  "  D1 = B(1);"
+  "}",
+  cxxOperatorCallExpr(hasArgument(
+  1, callExpr(hasArgument(0, ignoringElidableConstructorCall(
+ integerLiteral()),
+  LanguageMode::Cxx11OrLater));
+  EXPECT_TRUE(matches(
+  "struct H {};"
+  "H G();"
+  "void f() {"
+  "  H D = G();"
+  "}",
+  varDecl(hasInitializer(anyOf(
+  ignoringElidableConstructorCall(callExpr()),
+  exprWithCleanups(has(ignoringElidableConstructorCall(callExpr())),
+  LanguageMode::Cxx11OrLater));
+}
+
+TEST(Matc

[PATCH] D63149: Added AST matcher for ignoring elidable constructors

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom marked an inline comment as done.
jvikstrom added inline comments.



Comment at: clang/unittests/ASTMatchers/ASTMatchersTest.h:153
+  case LanguageMode::Cxx2aOrLater:
+LangModes = {LanguageMode::Cxx2a};
+  }

hokein wrote:
> nit: add a llvm_unreachable for `default`.
But the switch covers all enum values so that would give warnings


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D63149/new/

https://reviews.llvm.org/D63149



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D63253: [clang-tidy] Made abseil-faster-strsplit-delimiter tests pass on C++17

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom created this revision.
jvikstrom added a reviewer: hokein.
Herald added a subscriber: xazax.hun.
Herald added a project: clang.

Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D63253

Files:
  clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.cpp
  clang-tools-extra/test/clang-tidy/abseil-faster-strsplit-delimiter.cpp


Index: clang-tools-extra/test/clang-tidy/abseil-faster-strsplit-delimiter.cpp
===
--- clang-tools-extra/test/clang-tidy/abseil-faster-strsplit-delimiter.cpp
+++ clang-tools-extra/test/clang-tidy/abseil-faster-strsplit-delimiter.cpp
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy -std=c++11,c++14 %s abseil-faster-strsplit-delimiter 
%t
+// RUN: %check_clang_tidy -std=c++11-or-later %s 
abseil-faster-strsplit-delimiter %t
 // FIXME: Fix the checker to work in C++17 mode.
 
 namespace absl {
Index: clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.cpp
===
--- clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.cpp
+++ clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.cpp
@@ -20,21 +20,6 @@
 
 AST_MATCHER(StringLiteral, lengthIsOne) { return Node.getLength() == 1; }
 
-::internal::Matcher
-constructExprWithArg(llvm::StringRef ClassName,
- const ::internal::Matcher &Arg) {
-  auto ConstrExpr = cxxConstructExpr(hasType(recordDecl(hasName(ClassName))),
- hasArgument(0, ignoringParenCasts(Arg)));
-
-  return anyOf(ConstrExpr, cxxBindTemporaryExpr(has(ConstrExpr)));
-}
-
-::internal::Matcher
-copyConstructExprWithArg(llvm::StringRef ClassName,
- const ::internal::Matcher &Arg) {
-  return constructExprWithArg(ClassName, constructExprWithArg(ClassName, Arg));
-}
-
 llvm::Optional makeCharacterLiteral(const StringLiteral *Literal) 
{
   std::string Result;
   {
@@ -74,11 +59,17 @@
 
   // Binds to a string_view (either absl or std) that was passed by value and
   // contructed from string literal.
-  auto StringViewArg =
-  copyConstructExprWithArg("::absl::string_view", SingleChar);
+  auto StringViewArg = ignoringElidableConstructorCall(ignoringImpCasts(
+  cxxConstructExpr(hasType(recordDecl(hasName("::absl::string_view"))),
+   hasArgument(0, ignoringParenImpCasts(SingleChar);
 
+  // Need to ignore the elidable constructor as otherwise there is no match for
+  // c++14 and earlier.
   auto ByAnyCharArg =
-  expr(copyConstructExprWithArg("::absl::ByAnyChar", StringViewArg))
+  expr(has(ignoringElidableConstructorCall(
+   ignoringParenCasts(cxxBindTemporaryExpr(has(cxxConstructExpr(
+   hasType(recordDecl(hasName("::absl::ByAnyChar"))),
+   hasArgument(0, StringViewArg
   .bind("ByAnyChar");
 
   // Find uses of absl::StrSplit(..., "x") and absl::StrSplit(...,


Index: clang-tools-extra/test/clang-tidy/abseil-faster-strsplit-delimiter.cpp
===
--- clang-tools-extra/test/clang-tidy/abseil-faster-strsplit-delimiter.cpp
+++ clang-tools-extra/test/clang-tidy/abseil-faster-strsplit-delimiter.cpp
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy -std=c++11,c++14 %s abseil-faster-strsplit-delimiter %t
+// RUN: %check_clang_tidy -std=c++11-or-later %s abseil-faster-strsplit-delimiter %t
 // FIXME: Fix the checker to work in C++17 mode.
 
 namespace absl {
Index: clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.cpp
===
--- clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.cpp
+++ clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.cpp
@@ -20,21 +20,6 @@
 
 AST_MATCHER(StringLiteral, lengthIsOne) { return Node.getLength() == 1; }
 
-::internal::Matcher
-constructExprWithArg(llvm::StringRef ClassName,
- const ::internal::Matcher &Arg) {
-  auto ConstrExpr = cxxConstructExpr(hasType(recordDecl(hasName(ClassName))),
- hasArgument(0, ignoringParenCasts(Arg)));
-
-  return anyOf(ConstrExpr, cxxBindTemporaryExpr(has(ConstrExpr)));
-}
-
-::internal::Matcher
-copyConstructExprWithArg(llvm::StringRef ClassName,
- const ::internal::Matcher &Arg) {
-  return constructExprWithArg(ClassName, constructExprWithArg(ClassName, Arg));
-}
-
 llvm::Optional makeCharacterLiteral(const StringLiteral *Literal) {
   std::string Result;
   {
@@ -74,11 +59,17 @@
 
   // Binds to a string_view (either absl or std) that was passed by value and
   // contructed from string literal.
-  auto StringViewArg =
-  copyConstructExprWithArg("::absl::string_view", SingleChar);
+  auto StringViewArg = ignoringElidableConstructorCall(ignoringImpCasts(
+  cxxConstructExpr(hasType(recordDecl(hasName("::absl::string_view"

[PATCH] D63261: [clang-tidy] Fixed abseil-time-subtraction to work on C++17

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom created this revision.
jvikstrom added reviewers: hokein, gribozavr.
Herald added subscribers: cfe-commits, xazax.hun.
Herald added a project: clang.

Fixed abseil-time-subtraction to work on C++17


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D63261

Files:
  clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp
  clang-tools-extra/test/clang-tidy/abseil-time-subtraction.cpp


Index: clang-tools-extra/test/clang-tidy/abseil-time-subtraction.cpp
===
--- clang-tools-extra/test/clang-tidy/abseil-time-subtraction.cpp
+++ clang-tools-extra/test/clang-tidy/abseil-time-subtraction.cpp
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy -std=c++11,c++14 %s abseil-time-subtraction %t -- -- 
-I %S/Inputs
+// RUN: %check_clang_tidy -std=c++11-or-later %s abseil-time-subtraction %t -- 
-- -I %S/Inputs
 // FIXME: Fix the checker to work in C++17 mode.
 
 #include "absl/time/time.h"
Index: clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp
===
--- clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp
+++ clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp
@@ -29,16 +29,26 @@
 
 static bool isConstructorAssignment(const MatchFinder::MatchResult &Result,
 const Expr *Node) {
+  // For C++14 and earlier there are elidable constructors that must be matched
+  // in hasParent. The elidable constructors do not exist in C++17 and later 
and
+  // therefore an additional check that does not match against the elidable
+  // constructors are needed for this case.
   return selectFirst(
- "e", match(expr(hasParent(materializeTemporaryExpr(hasParent(
- cxxConstructExpr(hasParent(exprWithCleanups(
- hasParent(varDecl()
-.bind("e"),
-*Node, *Result.Context)) != nullptr;
+ "e",
+ match(callExpr(hasParent(materializeTemporaryExpr(
+hasParent(cxxConstructExpr(hasParent(
+
exprWithCleanups(hasParent(varDecl()
+   .bind("e"),
+   *Node, *Result.Context)) != nullptr ||
+ selectFirst("e",
+ 
match(callExpr(hasParent(varDecl())).bind("e"),
+   *Node, *Result.Context)) != nullptr;
 }
 
 static bool isArgument(const MatchFinder::MatchResult &Result,
const Expr *Node) {
+  // For the same reason as in isConstructorAssignment two separate 
selectFirsts
+  // need to be checked here
   return selectFirst(
  "e",
  match(expr(hasParent(
@@ -46,16 +56,26 @@
 hasParent(callExpr()),
 unless(hasParent(cxxOperatorCallExpr(
.bind("e"),
-   *Node, *Result.Context)) != nullptr;
+   *Node, *Result.Context)) != nullptr ||
+ selectFirst(
+ "e", match(expr(hasParent(callExpr()),
+ unless(hasParent(cxxOperatorCallExpr(
+.bind("e"),
+*Node, *Result.Context)) != nullptr;
 }
 
 static bool isReturn(const MatchFinder::MatchResult &Result, const Expr *Node) 
{
+  // For the same reason as in isConstructorAssignment two separate 
selectFirsts
+  // need to be checked here
   return selectFirst(
  "e", match(expr(hasParent(materializeTemporaryExpr(hasParent(
  cxxConstructExpr(hasParent(exprWithCleanups(
  hasParent(returnStmt()
 .bind("e"),
-*Node, *Result.Context)) != nullptr;
+*Node, *Result.Context)) != nullptr ||
+ selectFirst("e",
+ match(expr(hasParent(returnStmt())).bind("e"),
+   *Node, *Result.Context)) != nullptr;
 }
 
 static bool parensRequired(const MatchFinder::MatchResult &Result,


Index: clang-tools-extra/test/clang-tidy/abseil-time-subtraction.cpp
===
--- clang-tools-extra/test/clang-tidy/abseil-time-subtraction.cpp
+++ clang-tools-extra/test/clang-tidy/abseil-time-subtraction.cpp
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy -std=c++11,c++14 %s abseil-time-subtraction %t -- -- -I %S/Inputs
+// RUN: %check_clang_tidy -std=c++11-or-later %s abseil-time-subtraction %t -- -- -I %S/Inputs
 // FIXME: Fix the checker to work in C++17 mode.
 
 #include "absl/time/time.h"
Index: clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp
===
--- clang-tools-extra

[PATCH] D63262: [clang-tidy] Made abseil-upgrade-duration-conversions tests pass on c++17

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom created this revision.
jvikstrom added reviewers: hokein, gribozavr.
Herald added subscribers: cfe-commits, xazax.hun.
Herald added a project: clang.

Made abseil-upgrade-duration-conversions tests pass on c++17


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D63262

Files:
  clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp
  clang-tools-extra/test/clang-tidy/abseil-upgrade-duration-conversions.cpp


Index: clang-tools-extra/test/clang-tidy/abseil-upgrade-duration-conversions.cpp
===
--- clang-tools-extra/test/clang-tidy/abseil-upgrade-duration-conversions.cpp
+++ clang-tools-extra/test/clang-tidy/abseil-upgrade-duration-conversions.cpp
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy -std=c++11,c++14 %s 
abseil-upgrade-duration-conversions %t -- -- -I%S/Inputs
+// RUN: %check_clang_tidy -std=c++11-or-later %s 
abseil-upgrade-duration-conversions %t -- -- -I%S/Inputs
 // FIXME: Fix the checker to work in C++17 mode.
 
 using int64_t = long long;
Index: clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp
===
--- clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp
+++ clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp
@@ -40,7 +40,8 @@
   callee(functionDecl(
   hasParent(functionTemplateDecl()),
   unless(hasTemplateArgument(0, refersToType(builtinType(,
-  hasAnyName("operator*=", "operator/=",
+  hasAnyName("operator*=", "operator/="
+  .bind("OuterExpr"),
   this);
 
   // Match expressions like `a.operator*=(b)` and `a.operator/=(b)` where `a`
@@ -52,7 +53,8 @@
   hasParent(functionTemplateDecl()),
   unless(hasTemplateArgument(0, refersToType(builtinType(,
   hasAnyName("operator*=", "operator/="))),
-  argumentCountIs(1), hasArgument(0, expr().bind("arg"))),
+  argumentCountIs(1), hasArgument(0, expr().bind("arg")))
+  .bind("OuterExpr"),
   this);
 
   // Match expressions like `a * b`, `a / b`, `operator*(a, b)`, and
@@ -66,7 +68,8 @@
argumentCountIs(2),
hasArgument(0, expr(hasType(
   
cxxRecordDecl(hasName("::absl::Duration"),
-   hasArgument(1, expr().bind("arg"))),
+   hasArgument(1, expr().bind("arg")))
+  .bind("OuterExpr"),
   this);
 
   // Match expressions like `a * b` and `operator*(a, b)` where `a` is not of a
@@ -77,8 +80,9 @@
unless(hasTemplateArgument(0, refersToType(builtinType(,
hasName("::absl::operator*"))),
argumentCountIs(2), hasArgument(0, expr().bind("arg")),
-   hasArgument(1, expr(hasType(cxxRecordDecl(
-  hasName("::absl::Duration")),
+   hasArgument(1, expr(hasType(
+  
cxxRecordDecl(hasName("::absl::Duration"))
+  .bind("OuterExpr"),
   this);
 
   // For the factory functions, we match only the non-templated overloads that
@@ -103,8 +107,9 @@
 has(implicitCastExpr(hasCastKind(CK_UserDefinedConversion,
   hasParent(callExpr(
   callee(functionDecl(DurationFactoryFunction(),
-  unless(hasParent(functionTemplateDecl(),
-  hasArgument(0, expr().bind("arg"),
+  unless(hasParent(functionTemplateDecl(),
+  hasArgument(0, expr().bind("arg")
+  .bind("OuterExpr"),
   this);
 }
 
@@ -117,7 +122,10 @@
   const auto *ArgExpr = Result.Nodes.getNodeAs("arg");
   SourceLocation Loc = ArgExpr->getBeginLoc();
 
-  if (!match(isInTemplateInstantiation(), *ArgExpr, *Result.Context).empty()) {
+  const auto *OuterExpr = Result.Nodes.getNodeAs("OuterExpr");
+
+  if (!match(isInTemplateInstantiation(), *OuterExpr, *Result.Context)
+   .empty()) {
 if (MatchedTemplateLocations.count(Loc.getRawEncoding()) == 0) {
   // For each location matched in a template instantiation, we check if the
   // location can also be found in `MatchedTemplateLocations`. If it is not


Index: clang-tools-extra/test/clang-tidy/abseil-upgrade-duration-conversions.cpp
===
--- clang-tools-extra/test/clang-tidy/abseil-upgrade-duration-conversions.cpp
+++ clang-tools-extra/test/clang-tidy/abseil-upgrade-duration-conversions.cpp
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy -std=c++11,c++14 %s abseil-upgrade-duration-conversions %t -- -- -I%S/Inputs
+// RUN: %check_clang_tidy -std=c++11-or-later %s abseil-upgrade-duration-conversions %t -- -- -I%S/Inputs
 // FIXME: Fix the checker to work in C++17 mode.
 
 using int64_t = long long;
Index: clang-tools-extra/clang

[PATCH] D63263: [clang-tidy] Fixed abseil-duration-unnecessary-conversion tests for c++17

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom created this revision.
jvikstrom added reviewers: hokein, gribozavr.
Herald added subscribers: cfe-commits, xazax.hun.
Herald added a project: clang.

Fixed abseil-duration-unnecessary-conversion tests for c++17


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D63263

Files:
  clang-tools-extra/clang-tidy/abseil/DurationUnnecessaryConversionCheck.cpp
  clang-tools-extra/test/clang-tidy/abseil-duration-unnecessary-conversion.cpp


Index: 
clang-tools-extra/test/clang-tidy/abseil-duration-unnecessary-conversion.cpp
===
--- clang-tools-extra/test/clang-tidy/abseil-duration-unnecessary-conversion.cpp
+++ clang-tools-extra/test/clang-tidy/abseil-duration-unnecessary-conversion.cpp
@@ -1,5 +1,4 @@
-// RUN: %check_clang_tidy -std=c++11,c++14 %s 
abseil-duration-unnecessary-conversion %t -- -- -I %S/Inputs
-// FIXME: Fix the checker to work in C++17 mode.
+// RUN: %check_clang_tidy -std=c++11-or-later %s 
abseil-duration-unnecessary-conversion %t -- -- -I %S/Inputs
 
 #include "absl/time/time.h"
 
Index: 
clang-tools-extra/clang-tidy/abseil/DurationUnnecessaryConversionCheck.cpp
===
--- clang-tools-extra/clang-tidy/abseil/DurationUnnecessaryConversionCheck.cpp
+++ clang-tools-extra/clang-tidy/abseil/DurationUnnecessaryConversionCheck.cpp
@@ -30,10 +30,9 @@
 
 // Matcher which matches the current scale's factory with a `1` argument,
 // e.g. `absl::Seconds(1)`.
-auto factory_matcher = cxxConstructExpr(hasArgument(
-0,
+auto factory_matcher = ignoringElidableConstructorCall(
 callExpr(callee(functionDecl(hasName(DurationFactory))),
- hasArgument(0, 
ignoringImpCasts(integerLiteral(equals(1)));
+ hasArgument(0, ignoringImpCasts(integerLiteral(equals(1));
 
 // Matcher which matches either inverse function and binds its argument,
 // e.g. `absl::ToDoubleSeconds(dur)`.


Index: clang-tools-extra/test/clang-tidy/abseil-duration-unnecessary-conversion.cpp
===
--- clang-tools-extra/test/clang-tidy/abseil-duration-unnecessary-conversion.cpp
+++ clang-tools-extra/test/clang-tidy/abseil-duration-unnecessary-conversion.cpp
@@ -1,5 +1,4 @@
-// RUN: %check_clang_tidy -std=c++11,c++14 %s abseil-duration-unnecessary-conversion %t -- -- -I %S/Inputs
-// FIXME: Fix the checker to work in C++17 mode.
+// RUN: %check_clang_tidy -std=c++11-or-later %s abseil-duration-unnecessary-conversion %t -- -- -I %S/Inputs
 
 #include "absl/time/time.h"
 
Index: clang-tools-extra/clang-tidy/abseil/DurationUnnecessaryConversionCheck.cpp
===
--- clang-tools-extra/clang-tidy/abseil/DurationUnnecessaryConversionCheck.cpp
+++ clang-tools-extra/clang-tidy/abseil/DurationUnnecessaryConversionCheck.cpp
@@ -30,10 +30,9 @@
 
 // Matcher which matches the current scale's factory with a `1` argument,
 // e.g. `absl::Seconds(1)`.
-auto factory_matcher = cxxConstructExpr(hasArgument(
-0,
+auto factory_matcher = ignoringElidableConstructorCall(
 callExpr(callee(functionDecl(hasName(DurationFactory))),
- hasArgument(0, ignoringImpCasts(integerLiteral(equals(1)));
+ hasArgument(0, ignoringImpCasts(integerLiteral(equals(1));
 
 // Matcher which matches either inverse function and binds its argument,
 // e.g. `absl::ToDoubleSeconds(dur)`.
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D63127: [clang-tidy] Fixed checker for abseil to work in C++17 mode

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom abandoned this revision.
jvikstrom added a comment.

Resubmitted as 4 different CLs


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D63127/new/

https://reviews.llvm.org/D63127



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D63261: [clang-tidy] Fixed abseil-time-subtraction to work on C++17

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 204538.
jvikstrom marked an inline comment as done.
jvikstrom added a comment.

Using anyOf instead of multiple selectFirsts


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D63261/new/

https://reviews.llvm.org/D63261

Files:
  clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp
  clang-tools-extra/test/clang-tidy/abseil-time-subtraction.cpp


Index: clang-tools-extra/test/clang-tidy/abseil-time-subtraction.cpp
===
--- clang-tools-extra/test/clang-tidy/abseil-time-subtraction.cpp
+++ clang-tools-extra/test/clang-tidy/abseil-time-subtraction.cpp
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy -std=c++11,c++14 %s abseil-time-subtraction %t -- -- 
-I %S/Inputs
+// RUN: %check_clang_tidy -std=c++11-or-later %s abseil-time-subtraction %t -- 
-- -I %S/Inputs
 // FIXME: Fix the checker to work in C++17 mode.
 
 #include "absl/time/time.h"
Index: clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp
===
--- clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp
+++ clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp
@@ -29,33 +29,52 @@
 
 static bool isConstructorAssignment(const MatchFinder::MatchResult &Result,
 const Expr *Node) {
+  // For C++14 and earlier there are elidable constructors that must be matched
+  // in hasParent. The elidable constructors do not exist in C++17 and later 
and
+  // therefore an additional check that does not match against the elidable
+  // constructors are needed for this case.
   return selectFirst(
- "e", match(expr(hasParent(materializeTemporaryExpr(hasParent(
- cxxConstructExpr(hasParent(exprWithCleanups(
- hasParent(varDecl()
-.bind("e"),
-*Node, *Result.Context)) != nullptr;
+ "e",
+ match(expr(anyOf(
+   callExpr(hasParent(materializeTemporaryExpr(hasParent(
+
cxxConstructExpr(hasParent(exprWithCleanups(
+hasParent(varDecl()
+   .bind("e"),
+   callExpr(hasParent(varDecl())).bind("e"))),
+   *Node, *Result.Context)) != nullptr;
 }
 
 static bool isArgument(const MatchFinder::MatchResult &Result,
const Expr *Node) {
+  // For the same reason as in isConstructorAssignment two separate 
selectFirsts
+  // need to be checked here
   return selectFirst(
  "e",
- match(expr(hasParent(
-
materializeTemporaryExpr(hasParent(cxxConstructExpr(
-hasParent(callExpr()),
-unless(hasParent(cxxOperatorCallExpr(
-   .bind("e"),
-   *Node, *Result.Context)) != nullptr;
+ match(
+ expr(anyOf(
+ expr(hasParent(materializeTemporaryExpr(
+  hasParent(cxxConstructExpr(
+  hasParent(callExpr()),
+  unless(hasParent(cxxOperatorCallExpr(
+ .bind("e"),
+ expr(hasParent(callExpr()),
+  unless(hasParent(cxxOperatorCallExpr(
+ .bind("e"))),
+ *Node, *Result.Context)) != nullptr;
 }
 
 static bool isReturn(const MatchFinder::MatchResult &Result, const Expr *Node) 
{
+  // For the same reason as in isConstructorAssignment two separate 
selectFirsts
+  // need to be checked here
   return selectFirst(
- "e", match(expr(hasParent(materializeTemporaryExpr(hasParent(
- cxxConstructExpr(hasParent(exprWithCleanups(
- hasParent(returnStmt()
-.bind("e"),
-*Node, *Result.Context)) != nullptr;
+ "e",
+ match(expr(anyOf(
+   expr(hasParent(materializeTemporaryExpr(hasParent(
+cxxConstructExpr(hasParent(exprWithCleanups(
+hasParent(returnStmt()
+   .bind("e"),
+   expr(hasParent(returnStmt())).bind("e"))),
+   *Node, *Result.Context)) != nullptr;
 }
 
 static bool parensRequired(const MatchFinder::MatchResult &Result,


Index: clang-tools-extra/test/clang-tidy/abseil-time-subtraction.cpp
===
--- clang-tools-extra/test/clang-tidy/abseil-time-subtraction.cpp
+++ clang-tools-extra/test/clang-tidy/abseil-time-subtr

[PATCH] D63479: Added SemanticSymbolASTCollector for collecting semantic symbolsCurrently collects variable and function declarations

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom created this revision.
jvikstrom added a reviewer: hokein.
Herald added subscribers: cfe-commits, kadircet, arphaman, jkorous, 
ilya-biryukov, mgorny.
Herald added a project: clang.

[clangd] Added SemanticSymbolASTCollector that collects variable and function 
declarations


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D63479

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/SemanticSymbolASTCollector.cpp
  clang-tools-extra/clangd/SemanticSymbolASTCollector.h
  clang-tools-extra/clangd/unittests/CMakeLists.txt
  clang-tools-extra/clangd/unittests/SemanticSymbolASTCollectorTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticSymbolASTCollectorTests.cpp
===
--- /dev/null
+++ clang-tools-extra/clangd/unittests/SemanticSymbolASTCollectorTests.cpp
@@ -0,0 +1,64 @@
+//===-- SemanticSymbolASTCollectorTests.cpp - SemanticSymbolASTCollector tests
+//--*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+
+#include "Annotations.h"
+#include "ClangdUnit.h"
+#include "Protocol.h"
+#include "SemanticSymbolASTCollector.h"
+#include "SourceCode.h"
+#include "TestTU.h"
+#include "llvm/Support/ScopedPrinter.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+Position createPosition(int Line, int Character) {
+  Position Pos;
+  Pos.character = Character;
+  Pos.line = Line;
+  return Pos;
+}
+
+TEST(SemanticSymbolASTCollector, GetBeginningOfIdentifier) {
+  std::string Preamble = R"cpp(
+struct A {
+  double SomeMember;
+};
+void foo(int a) {
+  auto VeryLongVariableName = 12312;
+  A aa;
+}
+  )cpp";
+
+  Annotations TestCase(Preamble);
+  std::vector CorrectSymbols = std::vector{
+  SemanticSymbol(SemanticScope::FunctionDeclaration, createPosition(4, 9),
+ 3),
+  SemanticSymbol(SemanticScope::VariableDeclaration, createPosition(4, 17),
+ 1),
+  SemanticSymbol(SemanticScope::VariableDeclaration, createPosition(5, 11),
+ 20),
+  SemanticSymbol(SemanticScope::VariableDeclaration, createPosition(6, 12),
+ 2)};
+
+  auto AST = TestTU::withCode(TestCase.code()).build();
+  SemanticSymbolASTCollector Collector(AST.getASTContext());
+  Collector.TraverseAST(AST.getASTContext());
+  auto Symbols = Collector.getSymbols();
+  EXPECT_THAT(Symbols, ElementsAreArray(CorrectSymbols));
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
Index: clang-tools-extra/clangd/unittests/CMakeLists.txt
===
--- clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -53,6 +53,7 @@
   RenameTests.cpp
   RIFFTests.cpp
   SelectionTests.cpp
+  SemanticSymbolASTCollectorTests.cpp
   SerializationTests.cpp
   SourceCodeTests.cpp
   SymbolCollectorTests.cpp
Index: clang-tools-extra/clangd/SemanticSymbolASTCollector.h
===
--- /dev/null
+++ clang-tools-extra/clangd/SemanticSymbolASTCollector.h
@@ -0,0 +1,91 @@
+//===--- SemanticSymbolASTCollector.h - Manipulating source code as strings
+//-*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+//
+// Code for collecting semantic symbols from the C++ AST using the
+// RecursiveASTVisitor
+//
+//===--===//
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICSYMBOLASTCOLLECTOR_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICSYMBOLASTCOLLECTOR_H
+
+#include "AST.h"
+#include "Headers.h"
+#include "Protocol.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/Lex/Lexer.h"
+
+namespace clang {
+namespace clangd {
+
+// ScopeIndex represents the mapping from the scopes list to a type of
+// expression
+enum class SemanticScope : int {
+  VariableDeclaration = 0,
+  FunctionDeclaration = 1,
+};
+
+// Holds all information needed for the highlighting
+struct SemanticSymbol {
+  SemanticSymbol() {}
+  SemanticSymbol(SemanticScope Scope, Position StartPosition, unsigned int Len)
+  : Scope(Scope), StartPosition(StartPosition), Len(Len) {}
+  SemanticScope Scope;
+  Position StartPosition;
+  unsigned int Len;
+};
+
+bool operator==(const SemanticSymbol &Lhs, const SemanticSymb

[PATCH] D63479: Added SemanticSymbolASTCollector for collecting semantic symbolsCurrently collects variable and function declarations

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 205275.
jvikstrom added a comment.

Added dots at the end of comments

Updating D63479: Added SemanticSymbolASTCollector for collecting semantic 
symbols
=

Currently collects variable and function declarations


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D63479/new/

https://reviews.llvm.org/D63479

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/SemanticSymbolASTCollector.cpp
  clang-tools-extra/clangd/SemanticSymbolASTCollector.h
  clang-tools-extra/clangd/unittests/CMakeLists.txt
  clang-tools-extra/clangd/unittests/SemanticSymbolASTCollectorTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticSymbolASTCollectorTests.cpp
===
--- /dev/null
+++ clang-tools-extra/clangd/unittests/SemanticSymbolASTCollectorTests.cpp
@@ -0,0 +1,64 @@
+//===-- SemanticSymbolASTCollectorTests.cpp - SemanticSymbolASTCollector tests
+//--*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+
+#include "Annotations.h"
+#include "ClangdUnit.h"
+#include "Protocol.h"
+#include "SemanticSymbolASTCollector.h"
+#include "SourceCode.h"
+#include "TestTU.h"
+#include "llvm/Support/ScopedPrinter.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+Position createPosition(int Line, int Character) {
+  Position Pos;
+  Pos.character = Character;
+  Pos.line = Line;
+  return Pos;
+}
+
+TEST(SemanticSymbolASTCollector, GetBeginningOfIdentifier) {
+  std::string Preamble = R"cpp(
+struct A {
+  double SomeMember;
+};
+void foo(int a) {
+  auto VeryLongVariableName = 12312;
+  A aa;
+}
+  )cpp";
+
+  Annotations TestCase(Preamble);
+  std::vector CorrectSymbols = std::vector{
+  SemanticSymbol(SemanticScope::FunctionDeclaration, createPosition(4, 9),
+ 3),
+  SemanticSymbol(SemanticScope::VariableDeclaration, createPosition(4, 17),
+ 1),
+  SemanticSymbol(SemanticScope::VariableDeclaration, createPosition(5, 11),
+ 20),
+  SemanticSymbol(SemanticScope::VariableDeclaration, createPosition(6, 12),
+ 2)};
+
+  auto AST = TestTU::withCode(TestCase.code()).build();
+  SemanticSymbolASTCollector Collector(AST.getASTContext());
+  Collector.TraverseAST(AST.getASTContext());
+  auto Symbols = Collector.getSymbols();
+  EXPECT_THAT(Symbols, ElementsAreArray(CorrectSymbols));
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
Index: clang-tools-extra/clangd/unittests/CMakeLists.txt
===
--- clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -53,6 +53,7 @@
   RenameTests.cpp
   RIFFTests.cpp
   SelectionTests.cpp
+  SemanticSymbolASTCollectorTests.cpp
   SerializationTests.cpp
   SourceCodeTests.cpp
   SymbolCollectorTests.cpp
Index: clang-tools-extra/clangd/SemanticSymbolASTCollector.h
===
--- /dev/null
+++ clang-tools-extra/clangd/SemanticSymbolASTCollector.h
@@ -0,0 +1,91 @@
+//===--- SemanticSymbolASTCollector.h - Manipulating source code as strings
+//-*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+//
+// Code for collecting semantic symbols from the C++ AST using the
+// RecursiveASTVisitor
+//
+//===--===//
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICSYMBOLASTCOLLECTOR_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICSYMBOLASTCOLLECTOR_H
+
+#include "AST.h"
+#include "Headers.h"
+#include "Protocol.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/Lex/Lexer.h"
+
+namespace clang {
+namespace clangd {
+
+// ScopeIndex represents the mapping from the scopes list to a type of
+// expression.
+enum class SemanticScope : int {
+  VariableDeclaration = 0,
+  FunctionDeclaration = 1,
+};
+
+// Holds all information needed for the highlighting.
+struct SemanticSymbol {
+  SemanticSymbol() {}
+  SemanticSymbol(SemanticScope Scope, Position StartPosition, unsigned int Len)
+  : Scope(Scope), StartPosition(StartPosition), Len(Len) {}
+  SemanticScope Scope;
+ 

[PATCH] D63479: Added SemanticSymbolASTCollector for collecting semantic symbolsCurrently collects variable and function declarations

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 205301.
jvikstrom added a comment.

Moved SemanticSymbolASTVisitor to an anonymous namespace and added a function 
for getting highlights that returns highlights line by line

Updating D63479: Added SemanticSymbolASTCollector for collecting semantic 
symbols
=

Currently collects variable and function declarations


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D63479/new/

https://reviews.llvm.org/D63479

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/SemanticHighlight.cpp
  clang-tools-extra/clangd/SemanticHighlight.h
  clang-tools-extra/clangd/unittests/CMakeLists.txt
  clang-tools-extra/clangd/unittests/SemanticHighlightTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightTests.cpp
===
--- /dev/null
+++ clang-tools-extra/clangd/unittests/SemanticHighlightTests.cpp
@@ -0,0 +1,64 @@
+//===-- SemanticSymbolASTCollectorTests.cpp - SemanticSymbolASTCollector tests
+//--*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+
+#include "Annotations.h"
+#include "ClangdUnit.h"
+#include "Protocol.h"
+#include "SemanticHighlight.h"
+#include "SourceCode.h"
+#include "TestTU.h"
+#include "llvm/Support/ScopedPrinter.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+Position createPosition(int Line, int Character) {
+  Position Pos;
+  Pos.character = Character;
+  Pos.line = Line;
+  return Pos;
+}
+
+TEST(SemanticSymbolASTCollector, GetBeginningOfIdentifier) {
+  std::string Preamble = R"cpp(
+struct A {
+  double SomeMember;
+};
+void foo(int a) {
+  auto VeryLongVariableName = 12312;
+  A aa;
+}
+  )cpp";
+
+  Annotations TestCase(Preamble);
+  std::vector CorrectLines = std::vector{
+  LineHighlight(4, {SemanticSymbol(SemanticScope::FunctionDeclaration,
+   createPosition(4, 9), 3),
+SemanticSymbol(SemanticScope::VariableDeclaration,
+   createPosition(4, 17), 1)}),
+  LineHighlight(5, {SemanticSymbol(SemanticScope::VariableDeclaration,
+   createPosition(5, 11), 20)}),
+  LineHighlight(6, {SemanticSymbol(SemanticScope::VariableDeclaration,
+   createPosition(6, 12), 2)})
+
+  };
+
+  auto AST = TestTU::withCode(TestCase.code()).build();
+  auto Lines = getASTHighlights(AST.getASTContext());
+  EXPECT_THAT(Lines, ElementsAreArray(CorrectLines));
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
Index: clang-tools-extra/clangd/unittests/CMakeLists.txt
===
--- clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -53,6 +53,7 @@
   RenameTests.cpp
   RIFFTests.cpp
   SelectionTests.cpp
+  SemanticHighlightTests.cpp
   SerializationTests.cpp
   SourceCodeTests.cpp
   SymbolCollectorTests.cpp
Index: clang-tools-extra/clangd/SemanticHighlight.h
===
--- /dev/null
+++ clang-tools-extra/clangd/SemanticHighlight.h
@@ -0,0 +1,62 @@
+//===--- SemanticSymbolASTCollector.h - Manipulating source code as strings
+//-*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+//
+// Code for collecting semantic symbols from the C++ AST using the
+// RecursiveASTVisitor
+//
+//===--===//
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICSYMBOLASTCOLLECTOR_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICSYMBOLASTCOLLECTOR_H
+
+#include "AST.h"
+#include "Headers.h"
+#include "Protocol.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/Lex/Lexer.h"
+
+namespace clang {
+namespace clangd {
+
+// ScopeIndex represents the mapping from the scopes list to a type of
+// expression.
+enum class SemanticScope : int {
+  VariableDeclaration = 0,
+  FunctionDeclaration = 1,
+};
+
+// Contains all information needed for the highlighting a symbol.
+struct SemanticSymbol {
+  SemanticSymbol() {}
+  SemanticSymbol(SemanticScope Scope, Position StartPosition, unsigned int Len)
+

[PATCH] D63479: Added SemanticSymbolASTCollector for collecting semantic symbolsCurrently collects variable and function declarations

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom abandoned this revision.
jvikstrom added a comment.

Resubmitting later with "complete" prototype in CL


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D63479/new/

https://reviews.llvm.org/D63479



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D63559: [clang-tidy] Added functionality for getting semantic highlights for variable and function declarations

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom created this revision.
jvikstrom added reviewers: hokein, ilya-biryukov.
Herald added subscribers: cfe-commits, kadircet, arphaman, jkorous, xazax.hun.
Herald added a project: clang.

Adds functionality for getting semantic highlights


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D63559

Files:
  clang-tools-extra/clangd/SemanticHighlight.cpp
  clang-tools-extra/clangd/SemanticHighlight.h
  clang-tools-extra/clangd/unittests/SemanticHighlightTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightTests.cpp
===
--- /dev/null
+++ clang-tools-extra/clangd/unittests/SemanticHighlightTests.cpp
@@ -0,0 +1,76 @@
+//===-- SemanticSymbolASTCollectorTests.cpp - SemanticSymbolASTCollector tests
+//--*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+
+#include "Annotations.h"
+#include "ClangdUnit.h"
+#include "Protocol.h"
+#include "SemanticHighlight.h"
+#include "SourceCode.h"
+#include "TestTU.h"
+#include "llvm/Support/ScopedPrinter.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+Position createPosition(int Line, int Character) {
+  Position Pos;
+  Pos.character = Character;
+  Pos.line = Line;
+  return Pos;
+}
+
+TEST(SemanticSymbolASTCollector, GetBeginningOfIdentifier) {
+  std::string Preamble = R"cpp(
+struct A {
+  double SomeMember;
+};
+void $foo[[foo]](int $a[[a]]) {
+  auto $vlvn[[VeryLongVariableName]] = 12312;
+  A $aa[[aa]];
+}
+  )cpp";
+
+  Annotations Test(Preamble);
+  auto Foo = Test.range("foo");
+  auto A = Test.range("a");
+  auto VeryLong = Test.range("vlvn");
+  auto AA = Test.range("aa");
+  std::vector CorrectLines = std::vector{
+  LineHighlight(
+  Foo.start.line,
+  {SemanticSymbol(SemanticScope::FunctionDeclaration,
+  createPosition(Foo.start.line, Foo.start.character),
+  3),
+   SemanticSymbol(SemanticScope::VariableDeclaration,
+  createPosition(A.start.line, A.start.character), 1)}),
+  LineHighlight(
+  VeryLong.start.line,
+  {SemanticSymbol(
+  SemanticScope::VariableDeclaration,
+  createPosition(VeryLong.start.line, VeryLong.start.character),
+  VeryLong.end.character - VeryLong.start.character)}),
+  LineHighlight(
+  AA.start.line,
+  {SemanticSymbol(SemanticScope::VariableDeclaration,
+  createPosition(AA.start.line, AA.start.character),
+  2)})};
+
+  auto AST = TestTU::withCode(Test.code()).build();
+  auto Lines = getASTHighlights(AST.getASTContext());
+  EXPECT_THAT(Lines, ElementsAreArray(CorrectLines));
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
Index: clang-tools-extra/clangd/SemanticHighlight.h
===
--- /dev/null
+++ clang-tools-extra/clangd/SemanticHighlight.h
@@ -0,0 +1,67 @@
+//===--- SemanticHighlight.h - Generating highlights from the AST
+//-*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+//
+// Code for collecting semantic symbols from the C++ AST using the
+// RecursiveASTVisitor
+//
+//===--===//
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICSYMBOLASTCOLLECTOR_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICSYMBOLASTCOLLECTOR_H
+
+#include "AST.h"
+#include "Headers.h"
+#include "Protocol.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/Lex/Lexer.h"
+
+namespace clang {
+namespace clangd {
+
+// ScopeIndex represents the mapping from the scopes list to a type of
+// expression.
+enum class SemanticScope : int {
+  VariableDeclaration = 0,
+  FunctionDeclaration = 1,
+};
+
+// Contains all information needed for the highlighting a symbol.
+struct SemanticSymbol {
+  SemanticSymbol() {}
+  SemanticSymbol(SemanticScope Scope, Position StartPosition, unsigned int Len)
+  : Scope(Scope), StartPosition(StartPosition), Len(Len) {}
+  SemanticScope Scope;
+  Position StartPosition;
+  unsigned int Len;
+};
+
+bool operator==(const SemanticSymbol &Lhs, const SemanticSymbol &Rhs);
+bool operator!=(const SemanticSymbol &Lhs, const SemanticSymbol &Rhs);
+
+// Contains all highlights in a single line.
+st

[PATCH] D63559: [clang-tidy] Added functionality for getting semantic highlights for variable and function declarations

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 205608.
jvikstrom added a comment.
Herald added a subscriber: mgorny.

Adds CMakeLists changes


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D63559/new/

https://reviews.llvm.org/D63559

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/SemanticHighlight.cpp
  clang-tools-extra/clangd/SemanticHighlight.h
  clang-tools-extra/clangd/unittests/CMakeLists.txt
  clang-tools-extra/clangd/unittests/SemanticHighlightTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightTests.cpp
===
--- /dev/null
+++ clang-tools-extra/clangd/unittests/SemanticHighlightTests.cpp
@@ -0,0 +1,76 @@
+//===-- SemanticSymbolASTCollectorTests.cpp - SemanticSymbolASTCollector tests
+//--*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+
+#include "Annotations.h"
+#include "ClangdUnit.h"
+#include "Protocol.h"
+#include "SemanticHighlight.h"
+#include "SourceCode.h"
+#include "TestTU.h"
+#include "llvm/Support/ScopedPrinter.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+Position createPosition(int Line, int Character) {
+  Position Pos;
+  Pos.character = Character;
+  Pos.line = Line;
+  return Pos;
+}
+
+TEST(SemanticSymbolASTCollector, GetBeginningOfIdentifier) {
+  std::string Preamble = R"cpp(
+struct A {
+  double SomeMember;
+};
+void $foo[[foo]](int $a[[a]]) {
+  auto $vlvn[[VeryLongVariableName]] = 12312;
+  A $aa[[aa]];
+}
+  )cpp";
+
+  Annotations Test(Preamble);
+  auto Foo = Test.range("foo");
+  auto A = Test.range("a");
+  auto VeryLong = Test.range("vlvn");
+  auto AA = Test.range("aa");
+  std::vector CorrectLines = std::vector{
+  LineHighlight(
+  Foo.start.line,
+  {SemanticSymbol(SemanticScope::FunctionDeclaration,
+  createPosition(Foo.start.line, Foo.start.character),
+  3),
+   SemanticSymbol(SemanticScope::VariableDeclaration,
+  createPosition(A.start.line, A.start.character), 1)}),
+  LineHighlight(
+  VeryLong.start.line,
+  {SemanticSymbol(
+  SemanticScope::VariableDeclaration,
+  createPosition(VeryLong.start.line, VeryLong.start.character),
+  VeryLong.end.character - VeryLong.start.character)}),
+  LineHighlight(
+  AA.start.line,
+  {SemanticSymbol(SemanticScope::VariableDeclaration,
+  createPosition(AA.start.line, AA.start.character),
+  2)})};
+
+  auto AST = TestTU::withCode(Test.code()).build();
+  auto Lines = getASTHighlights(AST.getASTContext());
+  EXPECT_THAT(Lines, ElementsAreArray(CorrectLines));
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
Index: clang-tools-extra/clangd/unittests/CMakeLists.txt
===
--- clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -53,6 +53,7 @@
   RenameTests.cpp
   RIFFTests.cpp
   SelectionTests.cpp
+  SemanticHighlightTests.cpp
   SerializationTests.cpp
   SourceCodeTests.cpp
   SymbolCollectorTests.cpp
Index: clang-tools-extra/clangd/SemanticHighlight.h
===
--- /dev/null
+++ clang-tools-extra/clangd/SemanticHighlight.h
@@ -0,0 +1,67 @@
+//===--- SemanticHighlight.h - Generating highlights from the AST
+//-*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+//
+// Code for collecting semantic symbols from the C++ AST using the
+// RecursiveASTVisitor
+//
+//===--===//
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICSYMBOLASTCOLLECTOR_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICSYMBOLASTCOLLECTOR_H
+
+#include "AST.h"
+#include "Headers.h"
+#include "Protocol.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/Lex/Lexer.h"
+
+namespace clang {
+namespace clangd {
+
+// ScopeIndex represents the mapping from the scopes list to a type of
+// expression.
+enum class SemanticScope : int {
+  VariableDeclaration = 0,
+  FunctionDeclaration = 1,
+};
+
+// Contains all information needed for the highlighting a symbol.
+

[PATCH] D63559: [clang-tidy] Added functionality for getting semantic highlights for variable and function declarations

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 205626.
jvikstrom added a comment.

Renamed SemanticSymbol to SemanticToken


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D63559/new/

https://reviews.llvm.org/D63559

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/SemanticHighlight.cpp
  clang-tools-extra/clangd/SemanticHighlight.h
  clang-tools-extra/clangd/unittests/CMakeLists.txt
  clang-tools-extra/clangd/unittests/SemanticHighlightTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightTests.cpp
===
--- /dev/null
+++ clang-tools-extra/clangd/unittests/SemanticHighlightTests.cpp
@@ -0,0 +1,76 @@
+//===-- SemanticSymbolASTCollectorTests.cpp - SemanticSymbolASTCollector tests
+//--*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+
+#include "Annotations.h"
+#include "ClangdUnit.h"
+#include "Protocol.h"
+#include "SemanticHighlight.h"
+#include "SourceCode.h"
+#include "TestTU.h"
+#include "llvm/Support/ScopedPrinter.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+Position createPosition(int Line, int Character) {
+  Position Pos;
+  Pos.character = Character;
+  Pos.line = Line;
+  return Pos;
+}
+
+TEST(SemanticSymbolASTCollector, GetBeginningOfIdentifier) {
+  std::string Preamble = R"cpp(
+struct A {
+  double SomeMember;
+};
+void $foo[[foo]](int $a[[a]]) {
+  auto $vlvn[[VeryLongVariableName]] = 12312;
+  A $aa[[aa]];
+}
+  )cpp";
+
+  Annotations Test(Preamble);
+  auto Foo = Test.range("foo");
+  auto A = Test.range("a");
+  auto VeryLong = Test.range("vlvn");
+  auto AA = Test.range("aa");
+  std::vector CorrectLines = std::vector{
+  LineHighlight(
+  Foo.start.line,
+  {SemanticToken(SemanticScope::FunctionDeclaration,
+  createPosition(Foo.start.line, Foo.start.character),
+  3),
+   SemanticToken(SemanticScope::VariableDeclaration,
+  createPosition(A.start.line, A.start.character), 1)}),
+  LineHighlight(
+  VeryLong.start.line,
+  {SemanticToken(
+  SemanticScope::VariableDeclaration,
+  createPosition(VeryLong.start.line, VeryLong.start.character),
+  VeryLong.end.character - VeryLong.start.character)}),
+  LineHighlight(
+  AA.start.line,
+  {SemanticToken(SemanticScope::VariableDeclaration,
+  createPosition(AA.start.line, AA.start.character),
+  2)})};
+
+  auto AST = TestTU::withCode(Test.code()).build();
+  auto Lines = getASTHighlights(AST.getASTContext());
+  EXPECT_THAT(Lines, ElementsAreArray(CorrectLines));
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
Index: clang-tools-extra/clangd/unittests/CMakeLists.txt
===
--- clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -53,6 +53,7 @@
   RenameTests.cpp
   RIFFTests.cpp
   SelectionTests.cpp
+  SemanticHighlightTests.cpp
   SerializationTests.cpp
   SourceCodeTests.cpp
   SymbolCollectorTests.cpp
Index: clang-tools-extra/clangd/SemanticHighlight.h
===
--- /dev/null
+++ clang-tools-extra/clangd/SemanticHighlight.h
@@ -0,0 +1,67 @@
+//===--- SemanticHighlight.h - Generating highlights from the AST
+//-*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+//
+// Code for collecting semantic symbols from the C++ AST using the
+// RecursiveASTVisitor
+//
+//===--===//
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICSYMBOLASTCOLLECTOR_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICSYMBOLASTCOLLECTOR_H
+
+#include "AST.h"
+#include "Headers.h"
+#include "Protocol.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/Lex/Lexer.h"
+
+namespace clang {
+namespace clangd {
+
+// ScopeIndex represents the mapping from the scopes list to a type of
+// expression.
+enum class SemanticScope : int {
+  VariableDeclaration = 0,
+  FunctionDeclaration = 1,
+};
+
+// Contains all information needed for the highlighting a symbol.
+struct SemanticToken {

[PATCH] D64617: [clangd] Added highlighting for members and methods

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom created this revision.
jvikstrom added reviewers: hokein, sammccall, ilya-biryukov.
Herald added subscribers: cfe-commits, kadircet, arphaman, jkorous, MaskRay.
Herald added a project: clang.

Added highlighting for members and methods.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D64617

Files:
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/semantic-highlighting.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -37,7 +37,9 @@
   {HighlightingKind::Function, "Function"},
   {HighlightingKind::Class, "Class"},
   {HighlightingKind::Enum, "Enum"},
-  {HighlightingKind::Namespace, "Namespace"}};
+  {HighlightingKind::Namespace, "Namespace"},
+  {HighlightingKind::MemberVariable, "MemberVariable"},
+  {HighlightingKind::Method, "Method"}};
   std::vector ExpectedTokens;
   for (const auto &KindString : KindToString) {
 std::vector Toks = makeHighlightingTokens(
@@ -53,14 +55,14 @@
   const char *TestCases[] = {
 R"cpp(
   struct $Class[[AS]] {
-double SomeMember;
+double $MemberVariable[[SomeMember]];
   };
   struct {
   } $Variable[[S]];
   void $Function[[foo]](int $Variable[[A]], $Class[[AS]] $Variable[[As]]) {
 auto $Variable[[VeryLongVariableName]] = 12312;
 $Class[[AS]] $Variable[[AA]];
-auto $Variable[[L]] = $Variable[[AA]].SomeMember + $Variable[[A]];
+auto $Variable[[L]] = $Variable[[AA]].$MemberVariable[[SomeMember]] + $Variable[[A]];
 auto $Variable[[FN]] = [ $Variable[[AA]]](int $Variable[[A]]) -> void {};
 $Variable[[FN]](12312);
   }
@@ -72,19 +74,19 @@
 auto $Variable[[Bou]] = $Function[[Gah]];
   }
   struct $Class[[A]] {
-void $Function[[abc]]();
+void $Method[[abc]]();
   };
 )cpp",
 R"cpp(
   namespace $Namespace[[abc]] {
 template
 struct $Class[[A]] {
-  T t;
+  T $MemberVariable[[t]];
 };
   }
   template
   struct $Class[[C]] : $Namespace[[abc]]::A {
-typename T::A* D;
+typename T::A* $MemberVariable[[D]];
   };
   $Namespace[[abc]]::$Class[[A]] $Variable[[AA]];
   typedef $Namespace[[abc]]::$Class[[A]] AAA;
@@ -92,7 +94,7 @@
 $Class[[B]]();
 ~$Class[[B]]();
 void operator<<($Class[[B]]);
-$Class[[AAA]] AA;
+$Class[[AAA]] $MemberVariable[[AA]];
   };
   $Class[[B]]::$Class[[B]]() {}
   $Class[[B]]::~$Class[[B]]() {}
@@ -106,8 +108,8 @@
   enum class $Enum[[E]] {};
   enum $Enum[[EE]] {};
   struct $Class[[A]] {
-$Enum[[E]] EEE;
-$Enum[[EE]] ;
+$Enum[[E]] $MemberVariable[[EEE]];
+$Enum[[EE]] $MemberVariable[[]];
   };
 )cpp",
 R"cpp(
@@ -132,6 +134,27 @@
 $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::Hi;
   ::$Namespace[[vwz]]::$Class[[A]] $Variable[[B]];
   ::$Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[BB]];
+)cpp",
+R"cpp(
+  struct $Class[[D]] {
+double $MemberVariable[[C]];
+  };
+  struct $Class[[A]] {
+double $MemberVariable[[B]];
+$Class[[D]] $MemberVariable[[E]];
+void $Method[[foo]]() {
+  $MemberVariable[[B]] = 123;
+  this->$MemberVariable[[B]] = 156;
+  this->$Method[[foo]]();
+  $Method[[foo]]();
+}
+  };
+  void $Function[[foo]]() {
+$Class[[A]] $Variable[[AA]];
+$Variable[[AA]].$MemberVariable[[B]] += 2;
+$Variable[[AA]].$Method[[foo]]();
+$Variable[[AA]].$MemberVariable[[E]].$MemberVariable[[C]];
+  }
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
@@ -181,7 +204,7 @@
   toSemanticHighlightingInformation(Tokens);
   std::vector ExpectedResults = {
   {1, "AQAEAAA="},
-  {3, "CAAEAAAEAAMAAQ=="}};
+  {3, "CAAEAAAEAAMAAg=="}};
   EXPECT_EQ(ActualResults, ExpectedResults);
 }
 
Index: clang-tools-extra/clangd/test/semantic-highlighting.test
===
--- clang-tools-extra/clangd/test/semantic-highlighting.test
+++ clang-tools-extra/clangd/test/semantic-highlighting.test
@@ -5,12 +5,18 @@
 # CHECK:  "semanticHighlighting": {
 # CHECK-NEXT:"scopes": [
 # CHECK-NEXT:  [
-# CHECK-NEXT:"variable.cpp"
+# CHECK-NEXT:"variable.other.cpp"
+# CHECK-NEXT:  ],
+# CHECK-NEXT:  [
+# CHECK-NEXT:"variable.other.member.cpp"
 # CHECK-NEXT:  

[PATCH] D64624: [clangd] Added highlighting to enum constants.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom created this revision.
jvikstrom added reviewers: hokein, sammccall, ilya-biryukov.
Herald added subscribers: cfe-commits, kadircet, arphaman, jkorous, MaskRay.
Herald added a project: clang.

VSCode does not have a scope for enum constants. So they were placed under 
"constant.other.enum" as that seems to be the most correct scope for enum 
constants. However, this makes theia color them blue (the same color it uses 
for keywords).


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D64624

Files:
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/semantic-highlighting.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp


Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -37,7 +37,8 @@
   {HighlightingKind::Function, "Function"},
   {HighlightingKind::Class, "Class"},
   {HighlightingKind::Enum, "Enum"},
-  {HighlightingKind::Namespace, "Namespace"}};
+  {HighlightingKind::Namespace, "Namespace"},
+  {HighlightingKind::EnumConstant, "EnumConstant"}};
   std::vector ExpectedTokens;
   for (const auto &KindString : KindToString) {
 std::vector Toks = makeHighlightingTokens(
@@ -118,7 +119,7 @@
   namespace $Namespace[[cde]] {
 struct $Class[[A]] {
   enum class $Enum[[B]] {
-Hi,
+$EnumConstant[[Hi]],
   };
 };
   }
@@ -129,9 +130,20 @@
 $Namespace[[abc]]::$Namespace[[bcd]]::$Namespace[[cde]];
   $Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[AA]];
   $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]] $Variable[[AAA]] =
-$Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::Hi;
+$Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::$EnumConstant[[Hi]];
   ::$Namespace[[vwz]]::$Class[[A]] $Variable[[B]];
   ::$Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[BB]];
+)cpp",
+R"cpp(
+  enum $Enum[[ABC]] {
+$EnumConstant[[Hi]],
+  };
+  int $Variable[[I]] = $EnumConstant[[Hi]];
+  enum class $Enum[[BC]] {
+$EnumConstant[[A]],
+$EnumConstant[[B]],
+  };
+  $Enum[[BC]] $Variable[[L]] = $Enum[[BC]]::$EnumConstant[[B]];
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
Index: clang-tools-extra/clangd/test/semantic-highlighting.test
===
--- clang-tools-extra/clangd/test/semantic-highlighting.test
+++ clang-tools-extra/clangd/test/semantic-highlighting.test
@@ -17,6 +17,9 @@
 # CHECK-NEXT:"entity.name.type.enum.cpp"
 # CHECK-NEXT:  ],
 # CHECK-NEXT:  [
+# CHECK-NEXT:"constant.other.enum.cpp"
+# CHECK-NEXT:  ],
+# CHECK-NEXT:  [
 # CHECK-NEXT:"entity.name.namespace.cpp"
 # CHECK-NEXT:  ]
 # CHECK-NEXT:]
Index: clang-tools-extra/clangd/SemanticHighlighting.h
===
--- clang-tools-extra/clangd/SemanticHighlighting.h
+++ clang-tools-extra/clangd/SemanticHighlighting.h
@@ -28,6 +28,7 @@
   Function,
   Class,
   Enum,
+  EnumConstant,
   Namespace,
 
   NumKinds,
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -119,6 +119,10 @@
   addToken(Loc, HighlightingKind::Enum);
   return;
 }
+if(isa(D)) {
+  addToken(Loc, HighlightingKind::EnumConstant);
+  return;
+}
 if (isa(D)) {
   addToken(Loc, HighlightingKind::Variable);
   return;
@@ -249,6 +253,8 @@
 return "entity.name.type.class.cpp";
   case HighlightingKind::Enum:
 return "entity.name.type.enum.cpp";
+  case HighlightingKind::EnumConstant:
+return "constant.other.enum.cpp";
   case HighlightingKind::Namespace:
 return "entity.name.namespace.cpp";
   case HighlightingKind::NumKinds:


Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -37,7 +37,8 @@
   {HighlightingKind::Function, "Function"},
   {HighlightingKind::Class, "Class"},
   {HighlightingKind::Enum, "Enum"},
-  {HighlightingKind::Namespace, "Namespace"}};
+  {HighlightingKind::Namespace, "Namespace"},
+  {HighlightingKind::EnumConstant, "EnumConstant"}};
   std::vector ExpectedTokens;
   for (const a

[PATCH] D64634: [clangd] Fix duplicate highlighting tokens appearing in initializer lists

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom created this revision.
jvikstrom added reviewers: hokein, sammccall, ilya-biryukov.
Herald added subscribers: cfe-commits, kadircet, arphaman, mgrang, jkorous, 
MaskRay.
Herald added a project: clang.

The RecursiveASTVisitor sometimes visits exprs in initializer lists twice. 
Added deduplication to prevent duplicate highlighting tokens from appearing. 
Done by sorting and a linear search.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D64634

Files:
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp


Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -132,6 +132,13 @@
 $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::Hi;
   ::$Namespace[[vwz]]::$Class[[A]] $Variable[[B]];
   ::$Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[BB]];
+)cpp",
+R"cpp(
+  struct $Class[[AA]] {
+int A;
+  }
+  int $Variable[[B]];
+  $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
Index: clang-tools-extra/clangd/SemanticHighlighting.h
===
--- clang-tools-extra/clangd/SemanticHighlighting.h
+++ clang-tools-extra/clangd/SemanticHighlighting.h
@@ -40,6 +40,7 @@
 };
 
 bool operator==(const HighlightingToken &Lhs, const HighlightingToken &Rhs);
+bool operator<(const HighlightingToken &Lhs, const HighlightingToken &Rhs);
 
 // Returns all HighlightingTokens from an AST. Only generates highlights for 
the
 // main AST.
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -31,6 +31,16 @@
   std::vector collectTokens() {
 Tokens.clear();
 TraverseAST(Ctx);
+// Initializer lists can give duplicates of tokens, therefore all tokens
+// must be deduplicated.
+std::sort(Tokens.begin(), Tokens.end());
+for (unsigned I = 1; I < Tokens.size(); ++I) {
+  if (Tokens[I] == Tokens[I - 1]) {
+Tokens.erase(Tokens.begin() + I);
+--I;
+  }
+}
+
 return Tokens;
   }
 
@@ -70,7 +80,6 @@
 DeclarationName::Identifier)
   // Only want to highlight identifiers.
   return true;
-
 addToken(Ref->getLocation(), Ref->getDecl());
 return true;
   }
@@ -201,6 +210,10 @@
   return Lhs.Kind == Rhs.Kind && Lhs.R == Rhs.R;
 }
 
+bool operator<(const HighlightingToken &Lhs, const HighlightingToken &Rhs) {
+  return Lhs.R.start < Rhs.R.start;
+}
+
 std::vector getSemanticHighlightings(ParsedAST &AST) {
   return HighlightingTokenCollector(AST).collectTokens();
 }


Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -132,6 +132,13 @@
 $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::Hi;
   ::$Namespace[[vwz]]::$Class[[A]] $Variable[[B]];
   ::$Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[BB]];
+)cpp",
+R"cpp(
+  struct $Class[[AA]] {
+int A;
+  }
+  int $Variable[[B]];
+  $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
Index: clang-tools-extra/clangd/SemanticHighlighting.h
===
--- clang-tools-extra/clangd/SemanticHighlighting.h
+++ clang-tools-extra/clangd/SemanticHighlighting.h
@@ -40,6 +40,7 @@
 };
 
 bool operator==(const HighlightingToken &Lhs, const HighlightingToken &Rhs);
+bool operator<(const HighlightingToken &Lhs, const HighlightingToken &Rhs);
 
 // Returns all HighlightingTokens from an AST. Only generates highlights for the
 // main AST.
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -31,6 +31,16 @@
   std::vector collectTokens() {
 Tokens.clear();
 TraverseAST(Ctx);
+// Initializer lists can give duplicates of tokens, therefore all tokens
+// must be deduplicated.
+std::sort(Tokens.begin(), Tokens.end());
+for (unsigned I = 1; I < Tokens.size(); ++I) {
+  if (Tokens[I] == Tokens[I - 1]) {
+Tokens.erase(Tokens.begin() + I);
+--I;
+  }
+}
+
 retur

[PATCH] D64617: [clangd] Added highlighting for members and methods

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 209510.
jvikstrom added a comment.

Removed addToken for Exprs.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64617/new/

https://reviews.llvm.org/D64617

Files:
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/semantic-highlighting.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -37,7 +37,9 @@
   {HighlightingKind::Function, "Function"},
   {HighlightingKind::Class, "Class"},
   {HighlightingKind::Enum, "Enum"},
-  {HighlightingKind::Namespace, "Namespace"}};
+  {HighlightingKind::Namespace, "Namespace"},
+  {HighlightingKind::Field, "Field"},
+  {HighlightingKind::Method, "Method"}};
   std::vector ExpectedTokens;
   for (const auto &KindString : KindToString) {
 std::vector Toks = makeHighlightingTokens(
@@ -53,14 +55,14 @@
   const char *TestCases[] = {
 R"cpp(
   struct $Class[[AS]] {
-double SomeMember;
+double $Field[[SomeMember]];
   };
   struct {
   } $Variable[[S]];
   void $Function[[foo]](int $Variable[[A]], $Class[[AS]] $Variable[[As]]) {
 auto $Variable[[VeryLongVariableName]] = 12312;
 $Class[[AS]] $Variable[[AA]];
-auto $Variable[[L]] = $Variable[[AA]].SomeMember + $Variable[[A]];
+auto $Variable[[L]] = $Variable[[AA]].$Field[[SomeMember]] + $Variable[[A]];
 auto $Variable[[FN]] = [ $Variable[[AA]]](int $Variable[[A]]) -> void {};
 $Variable[[FN]](12312);
   }
@@ -72,19 +74,19 @@
 auto $Variable[[Bou]] = $Function[[Gah]];
   }
   struct $Class[[A]] {
-void $Function[[abc]]();
+void $Method[[abc]]();
   };
 )cpp",
 R"cpp(
   namespace $Namespace[[abc]] {
 template
 struct $Class[[A]] {
-  T t;
+  T $Field[[t]];
 };
   }
   template
   struct $Class[[C]] : $Namespace[[abc]]::A {
-typename T::A* D;
+typename T::A* $Field[[D]];
   };
   $Namespace[[abc]]::$Class[[A]] $Variable[[AA]];
   typedef $Namespace[[abc]]::$Class[[A]] AAA;
@@ -92,7 +94,7 @@
 $Class[[B]]();
 ~$Class[[B]]();
 void operator<<($Class[[B]]);
-$Class[[AAA]] AA;
+$Class[[AAA]] $Field[[AA]];
   };
   $Class[[B]]::$Class[[B]]() {}
   $Class[[B]]::~$Class[[B]]() {}
@@ -106,8 +108,8 @@
   enum class $Enum[[E]] {};
   enum $Enum[[EE]] {};
   struct $Class[[A]] {
-$Enum[[E]] EEE;
-$Enum[[EE]] ;
+$Enum[[E]] $Field[[EEE]];
+$Enum[[EE]] $Field[[]];
   };
 )cpp",
 R"cpp(
@@ -132,6 +134,30 @@
 $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::Hi;
   ::$Namespace[[vwz]]::$Class[[A]] $Variable[[B]];
   ::$Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[BB]];
+)cpp",
+R"cpp(
+  struct $Class[[D]] {
+double $Field[[C]];
+  };
+  struct $Class[[A]] {
+double $Field[[B]];
+$Class[[D]] $Field[[E]];
+static double $Variable[[S]];
+void $Method[[foo]]() {
+  $Field[[B]] = 123;
+  this->$Field[[B]] = 156;
+  this->$Method[[foo]]();
+  $Method[[foo]]();
+  $Variable[[S]] = 90.1;
+}
+  };
+  void $Function[[foo]]() {
+$Class[[A]] $Variable[[AA]];
+$Variable[[AA]].$Field[[B]] += 2;
+$Variable[[AA]].$Method[[foo]]();
+$Variable[[AA]].$Field[[E]].$Field[[C]];
+$Class[[A]]::$Variable[[S]] = 90;
+  }
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
@@ -181,7 +207,7 @@
   toSemanticHighlightingInformation(Tokens);
   std::vector ExpectedResults = {
   {1, "AQAEAAA="},
-  {3, "CAAEAAAEAAMAAQ=="}};
+  {3, "CAAEAAAEAAMAAg=="}};
   EXPECT_EQ(ActualResults, ExpectedResults);
 }
 
Index: clang-tools-extra/clangd/test/semantic-highlighting.test
===
--- clang-tools-extra/clangd/test/semantic-highlighting.test
+++ clang-tools-extra/clangd/test/semantic-highlighting.test
@@ -5,12 +5,18 @@
 # CHECK:  "semanticHighlighting": {
 # CHECK-NEXT:"scopes": [
 # CHECK-NEXT:  [
-# CHECK-NEXT:"variable.cpp"
+# CHECK-NEXT:"variable.other.cpp"
+# CHECK-NEXT:  ],
+# CHECK-NEXT:  [
+# CHECK-NEXT:"variable.other.field.cpp"
 # CHECK-NEXT:  ],
 # CHECK-NEXT:  [
 # CHECK-NEXT:"entity.name.function.cpp"
 # CHECK-NEXT:  ]

[PATCH] D64617: [clangd] Added highlighting for members and methods

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 209509.
jvikstrom marked 6 inline comments as done.
jvikstrom added a comment.

Addressed comments.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64617/new/

https://reviews.llvm.org/D64617

Files:
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/semantic-highlighting.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -37,7 +37,9 @@
   {HighlightingKind::Function, "Function"},
   {HighlightingKind::Class, "Class"},
   {HighlightingKind::Enum, "Enum"},
-  {HighlightingKind::Namespace, "Namespace"}};
+  {HighlightingKind::Namespace, "Namespace"},
+  {HighlightingKind::Field, "Field"},
+  {HighlightingKind::Method, "Method"}};
   std::vector ExpectedTokens;
   for (const auto &KindString : KindToString) {
 std::vector Toks = makeHighlightingTokens(
@@ -53,14 +55,14 @@
   const char *TestCases[] = {
 R"cpp(
   struct $Class[[AS]] {
-double SomeMember;
+double $Field[[SomeMember]];
   };
   struct {
   } $Variable[[S]];
   void $Function[[foo]](int $Variable[[A]], $Class[[AS]] $Variable[[As]]) {
 auto $Variable[[VeryLongVariableName]] = 12312;
 $Class[[AS]] $Variable[[AA]];
-auto $Variable[[L]] = $Variable[[AA]].SomeMember + $Variable[[A]];
+auto $Variable[[L]] = $Variable[[AA]].$Field[[SomeMember]] + $Variable[[A]];
 auto $Variable[[FN]] = [ $Variable[[AA]]](int $Variable[[A]]) -> void {};
 $Variable[[FN]](12312);
   }
@@ -72,19 +74,19 @@
 auto $Variable[[Bou]] = $Function[[Gah]];
   }
   struct $Class[[A]] {
-void $Function[[abc]]();
+void $Method[[abc]]();
   };
 )cpp",
 R"cpp(
   namespace $Namespace[[abc]] {
 template
 struct $Class[[A]] {
-  T t;
+  T $Field[[t]];
 };
   }
   template
   struct $Class[[C]] : $Namespace[[abc]]::A {
-typename T::A* D;
+typename T::A* $Field[[D]];
   };
   $Namespace[[abc]]::$Class[[A]] $Variable[[AA]];
   typedef $Namespace[[abc]]::$Class[[A]] AAA;
@@ -92,7 +94,7 @@
 $Class[[B]]();
 ~$Class[[B]]();
 void operator<<($Class[[B]]);
-$Class[[AAA]] AA;
+$Class[[AAA]] $Field[[AA]];
   };
   $Class[[B]]::$Class[[B]]() {}
   $Class[[B]]::~$Class[[B]]() {}
@@ -106,8 +108,8 @@
   enum class $Enum[[E]] {};
   enum $Enum[[EE]] {};
   struct $Class[[A]] {
-$Enum[[E]] EEE;
-$Enum[[EE]] ;
+$Enum[[E]] $Field[[EEE]];
+$Enum[[EE]] $Field[[]];
   };
 )cpp",
 R"cpp(
@@ -132,6 +134,30 @@
 $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::Hi;
   ::$Namespace[[vwz]]::$Class[[A]] $Variable[[B]];
   ::$Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[BB]];
+)cpp",
+R"cpp(
+  struct $Class[[D]] {
+double $Field[[C]];
+  };
+  struct $Class[[A]] {
+double $Field[[B]];
+$Class[[D]] $Field[[E]];
+static double $Variable[[S]];
+void $Method[[foo]]() {
+  $Field[[B]] = 123;
+  this->$Field[[B]] = 156;
+  this->$Method[[foo]]();
+  $Method[[foo]]();
+  $Variable[[S]] = 90.1;
+}
+  };
+  void $Function[[foo]]() {
+$Class[[A]] $Variable[[AA]];
+$Variable[[AA]].$Field[[B]] += 2;
+$Variable[[AA]].$Method[[foo]]();
+$Variable[[AA]].$Field[[E]].$Field[[C]];
+$Class[[A]]::$Variable[[S]] = 90;
+  }
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
@@ -181,7 +207,7 @@
   toSemanticHighlightingInformation(Tokens);
   std::vector ExpectedResults = {
   {1, "AQAEAAA="},
-  {3, "CAAEAAAEAAMAAQ=="}};
+  {3, "CAAEAAAEAAMAAg=="}};
   EXPECT_EQ(ActualResults, ExpectedResults);
 }
 
Index: clang-tools-extra/clangd/test/semantic-highlighting.test
===
--- clang-tools-extra/clangd/test/semantic-highlighting.test
+++ clang-tools-extra/clangd/test/semantic-highlighting.test
@@ -5,12 +5,18 @@
 # CHECK:  "semanticHighlighting": {
 # CHECK-NEXT:"scopes": [
 # CHECK-NEXT:  [
-# CHECK-NEXT:"variable.cpp"
+# CHECK-NEXT:"variable.other.cpp"
+# CHECK-NEXT:  ],
+# CHECK-NEXT:  [
+# CHECK-NEXT:"variable.other.field.cpp"
 # CHECK-NEXT:  ],
 # CHECK-NEXT:  [
 # CHECK-NEXT:"entity.name.fun

[PATCH] D64624: [clangd] Added highlighting to enum constants.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 209514.
jvikstrom marked 2 inline comments as done.
jvikstrom added a comment.

Address comments.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64624/new/

https://reviews.llvm.org/D64624

Files:
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/semantic-highlighting.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -37,7 +37,8 @@
   {HighlightingKind::Function, "Function"},
   {HighlightingKind::Class, "Class"},
   {HighlightingKind::Enum, "Enum"},
-  {HighlightingKind::Namespace, "Namespace"}};
+  {HighlightingKind::Namespace, "Namespace"},
+  {HighlightingKind::EnumConstant, "EnumConstant"}};
   std::vector ExpectedTokens;
   for (const auto &KindString : KindToString) {
 std::vector Toks = makeHighlightingTokens(
@@ -103,12 +104,19 @@
   }
 )cpp",
 R"cpp(
-  enum class $Enum[[E]] {};
-  enum $Enum[[EE]] {};
+  enum class $Enum[[E]] {
+$EnumConstant[[A]],
+$EnumConstant[[B]],
+  };
+  enum $Enum[[EE]] {
+$EnumConstant[[Hi]],
+  };
   struct $Class[[A]] {
 $Enum[[E]] EEE;
 $Enum[[EE]] ;
   };
+  int $Variable[[I]] = $EnumConstant[[Hi]];
+  $Enum[[E]] $Variable[[L]] = $Enum[[E]]::$EnumConstant[[B]];
 )cpp",
 R"cpp(
   namespace $Namespace[[abc]] {
@@ -118,7 +126,7 @@
   namespace $Namespace[[cde]] {
 struct $Class[[A]] {
   enum class $Enum[[B]] {
-Hi,
+$EnumConstant[[Hi]],
   };
 };
   }
@@ -129,7 +137,7 @@
 $Namespace[[abc]]::$Namespace[[bcd]]::$Namespace[[cde]];
   $Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[AA]];
   $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]] $Variable[[AAA]] =
-$Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::Hi;
+$Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::$EnumConstant[[Hi]];
   ::$Namespace[[vwz]]::$Class[[A]] $Variable[[B]];
   ::$Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[BB]];
 )cpp"};
Index: clang-tools-extra/clangd/test/semantic-highlighting.test
===
--- clang-tools-extra/clangd/test/semantic-highlighting.test
+++ clang-tools-extra/clangd/test/semantic-highlighting.test
@@ -17,6 +17,9 @@
 # CHECK-NEXT:"entity.name.type.enum.cpp"
 # CHECK-NEXT:  ],
 # CHECK-NEXT:  [
+# CHECK-NEXT:"constant.other.enum.cpp"
+# CHECK-NEXT:  ],
+# CHECK-NEXT:  [
 # CHECK-NEXT:"entity.name.namespace.cpp"
 # CHECK-NEXT:  ]
 # CHECK-NEXT:]
Index: clang-tools-extra/clangd/SemanticHighlighting.h
===
--- clang-tools-extra/clangd/SemanticHighlighting.h
+++ clang-tools-extra/clangd/SemanticHighlighting.h
@@ -28,6 +28,7 @@
   Function,
   Class,
   Enum,
+  EnumConstant,
   Namespace,
 
   NumKinds,
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -119,6 +119,10 @@
   addToken(Loc, HighlightingKind::Enum);
   return;
 }
+if (isa(D)) {
+  addToken(Loc, HighlightingKind::EnumConstant);
+  return;
+}
 if (isa(D)) {
   addToken(Loc, HighlightingKind::Variable);
   return;
@@ -249,6 +253,8 @@
 return "entity.name.type.class.cpp";
   case HighlightingKind::Enum:
 return "entity.name.type.enum.cpp";
+  case HighlightingKind::EnumConstant:
+return "constant.other.enum.cpp";
   case HighlightingKind::Namespace:
 return "entity.name.namespace.cpp";
   case HighlightingKind::NumKinds:
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64624: [clangd] Added highlighting to enum constants.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom marked an inline comment as done.
jvikstrom added inline comments.



Comment at: clang-tools-extra/clangd/SemanticHighlighting.cpp:257
+  case HighlightingKind::EnumConstant:
+return "constant.other.enum.cpp";
   case HighlightingKind::Namespace:

hokein wrote:
> hokein wrote:
> > could you check the tm scope on vscode? They seem to use 
> > `variable.other.enummember`.
> this comment is undone, vscode is using 
> https://github.com/microsoft/vscode/blob/master/extensions/cpp/syntaxes/cpp.tmLanguage.json#L10024
Oh yeah, I was going to write a comment about keeping it as 
`constant.other.enum` or something similar to it. Because it's not really a 
variable. But as we've stuck to the same TM scopes as vscode we should probably 
keep doing that so I'll change to `variable.other.enummember` and land.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64624/new/

https://reviews.llvm.org/D64624



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64624: [clangd] Added highlighting to enum constants.

[Johan Vikström via Phabricator via cfe-commits]

This revision was automatically updated to reflect the committed changes.
Closed by commit rL366045: [clangd] Added highlighting to enum constants. 
(authored by jvikstrom, committed by ).
Herald added a project: LLVM.
Herald added a subscriber: llvm-commits.

Changed prior to commit:
  https://reviews.llvm.org/D64624?vs=209514&id=209769#toc

Repository:
  rL LLVM

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64624/new/

https://reviews.llvm.org/D64624

Files:
  clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
  clang-tools-extra/trunk/clangd/SemanticHighlighting.h
  clang-tools-extra/trunk/clangd/test/semantic-highlighting.test
  clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
@@ -119,6 +119,10 @@
   addToken(Loc, HighlightingKind::Enum);
   return;
 }
+if (isa(D)) {
+  addToken(Loc, HighlightingKind::EnumConstant);
+  return;
+}
 if (isa(D)) {
   addToken(Loc, HighlightingKind::Variable);
   return;
@@ -249,6 +253,8 @@
 return "entity.name.type.class.cpp";
   case HighlightingKind::Enum:
 return "entity.name.type.enum.cpp";
+  case HighlightingKind::EnumConstant:
+return "variable.other.enummember.cpp";
   case HighlightingKind::Namespace:
 return "entity.name.namespace.cpp";
   case HighlightingKind::NumKinds:
Index: clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
@@ -37,7 +37,8 @@
   {HighlightingKind::Function, "Function"},
   {HighlightingKind::Class, "Class"},
   {HighlightingKind::Enum, "Enum"},
-  {HighlightingKind::Namespace, "Namespace"}};
+  {HighlightingKind::Namespace, "Namespace"},
+  {HighlightingKind::EnumConstant, "EnumConstant"}};
   std::vector ExpectedTokens;
   for (const auto &KindString : KindToString) {
 std::vector Toks = makeHighlightingTokens(
@@ -103,12 +104,19 @@
   }
 )cpp",
 R"cpp(
-  enum class $Enum[[E]] {};
-  enum $Enum[[EE]] {};
+  enum class $Enum[[E]] {
+$EnumConstant[[A]],
+$EnumConstant[[B]],
+  };
+  enum $Enum[[EE]] {
+$EnumConstant[[Hi]],
+  };
   struct $Class[[A]] {
 $Enum[[E]] EEE;
 $Enum[[EE]] ;
   };
+  int $Variable[[I]] = $EnumConstant[[Hi]];
+  $Enum[[E]] $Variable[[L]] = $Enum[[E]]::$EnumConstant[[B]];
 )cpp",
 R"cpp(
   namespace $Namespace[[abc]] {
@@ -118,7 +126,7 @@
   namespace $Namespace[[cde]] {
 struct $Class[[A]] {
   enum class $Enum[[B]] {
-Hi,
+$EnumConstant[[Hi]],
   };
 };
   }
@@ -129,7 +137,7 @@
 $Namespace[[abc]]::$Namespace[[bcd]]::$Namespace[[cde]];
   $Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[AA]];
   $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]] $Variable[[AAA]] =
-$Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::Hi;
+$Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::$EnumConstant[[Hi]];
   ::$Namespace[[vwz]]::$Class[[A]] $Variable[[B]];
   ::$Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[BB]];
 )cpp"};
Index: clang-tools-extra/trunk/clangd/SemanticHighlighting.h
===
--- clang-tools-extra/trunk/clangd/SemanticHighlighting.h
+++ clang-tools-extra/trunk/clangd/SemanticHighlighting.h
@@ -28,6 +28,7 @@
   Function,
   Class,
   Enum,
+  EnumConstant,
   Namespace,
 
   NumKinds,
Index: clang-tools-extra/trunk/clangd/test/semantic-highlighting.test
===
--- clang-tools-extra/trunk/clangd/test/semantic-highlighting.test
+++ clang-tools-extra/trunk/clangd/test/semantic-highlighting.test
@@ -17,6 +17,9 @@
 # CHECK-NEXT:"entity.name.type.enum.cpp"
 # CHECK-NEXT:  ],
 # CHECK-NEXT:  [
+# CHECK-NEXT:"variable.other.enummember.cpp"
+# CHECK-NEXT:  ],
+# CHECK-NEXT:  [
 # CHECK-NEXT:"entity.name.namespace.cpp"
 # CHECK-NEXT:  ]
 # CHECK-NEXT:]
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64617: [clangd] Added highlighting for members and methods

[Johan Vikström via Phabricator via cfe-commits]

This revision was automatically updated to reflect the committed changes.
Closed by commit rL366047: [clangd] Added highlighting for members and methods. 
(authored by jvikstrom, committed by ).
Herald added a project: LLVM.
Herald added a subscriber: llvm-commits.

Changed prior to commit:
  https://reviews.llvm.org/D64617?vs=209510&id=209770#toc

Repository:
  rL LLVM

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64617/new/

https://reviews.llvm.org/D64617

Files:
  clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
  clang-tools-extra/trunk/clangd/SemanticHighlighting.h
  clang-tools-extra/trunk/clangd/test/semantic-highlighting.test
  clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/trunk/clangd/test/semantic-highlighting.test
===
--- clang-tools-extra/trunk/clangd/test/semantic-highlighting.test
+++ clang-tools-extra/trunk/clangd/test/semantic-highlighting.test
@@ -5,12 +5,18 @@
 # CHECK:  "semanticHighlighting": {
 # CHECK-NEXT:"scopes": [
 # CHECK-NEXT:  [
-# CHECK-NEXT:"variable.cpp"
+# CHECK-NEXT:"variable.other.cpp"
 # CHECK-NEXT:  ],
 # CHECK-NEXT:  [
 # CHECK-NEXT:"entity.name.function.cpp"
 # CHECK-NEXT:  ],
 # CHECK-NEXT:  [
+# CHECK-NEXT:"entity.name.function.method.cpp"
+# CHECK-NEXT:  ],
+# CHECK-NEXT:  [
+# CHECK-NEXT:"variable.other.field.cpp"
+# CHECK-NEXT:  ],
+# CHECK-NEXT:  [
 # CHECK-NEXT:"entity.name.type.class.cpp"
 # CHECK-NEXT:  ],
 # CHECK-NEXT:  [
Index: clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
@@ -38,7 +38,9 @@
   {HighlightingKind::Class, "Class"},
   {HighlightingKind::Enum, "Enum"},
   {HighlightingKind::Namespace, "Namespace"},
-  {HighlightingKind::EnumConstant, "EnumConstant"}};
+  {HighlightingKind::EnumConstant, "EnumConstant"},
+  {HighlightingKind::Field, "Field"},
+  {HighlightingKind::Method, "Method"}};
   std::vector ExpectedTokens;
   for (const auto &KindString : KindToString) {
 std::vector Toks = makeHighlightingTokens(
@@ -54,14 +56,14 @@
   const char *TestCases[] = {
 R"cpp(
   struct $Class[[AS]] {
-double SomeMember;
+double $Field[[SomeMember]];
   };
   struct {
   } $Variable[[S]];
   void $Function[[foo]](int $Variable[[A]], $Class[[AS]] $Variable[[As]]) {
 auto $Variable[[VeryLongVariableName]] = 12312;
 $Class[[AS]] $Variable[[AA]];
-auto $Variable[[L]] = $Variable[[AA]].SomeMember + $Variable[[A]];
+auto $Variable[[L]] = $Variable[[AA]].$Field[[SomeMember]] + $Variable[[A]];
 auto $Variable[[FN]] = [ $Variable[[AA]]](int $Variable[[A]]) -> void {};
 $Variable[[FN]](12312);
   }
@@ -73,19 +75,19 @@
 auto $Variable[[Bou]] = $Function[[Gah]];
   }
   struct $Class[[A]] {
-void $Function[[abc]]();
+void $Method[[abc]]();
   };
 )cpp",
 R"cpp(
   namespace $Namespace[[abc]] {
 template
 struct $Class[[A]] {
-  T t;
+  T $Field[[t]];
 };
   }
   template
   struct $Class[[C]] : $Namespace[[abc]]::A {
-typename T::A* D;
+typename T::A* $Field[[D]];
   };
   $Namespace[[abc]]::$Class[[A]] $Variable[[AA]];
   typedef $Namespace[[abc]]::$Class[[A]] AAA;
@@ -93,7 +95,7 @@
 $Class[[B]]();
 ~$Class[[B]]();
 void operator<<($Class[[B]]);
-$Class[[AAA]] AA;
+$Class[[AAA]] $Field[[AA]];
   };
   $Class[[B]]::$Class[[B]]() {}
   $Class[[B]]::~$Class[[B]]() {}
@@ -112,8 +114,8 @@
 $EnumConstant[[Hi]],
   };
   struct $Class[[A]] {
-$Enum[[E]] EEE;
-$Enum[[EE]] ;
+$Enum[[E]] $Field[[EEE]];
+$Enum[[EE]] $Field[[]];
   };
   int $Variable[[I]] = $EnumConstant[[Hi]];
   $Enum[[E]] $Variable[[L]] = $Enum[[E]]::$EnumConstant[[B]];
@@ -140,6 +142,30 @@
 $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::$EnumConstant[[Hi]];
   ::$Namespace[[vwz]]::$Class[[A]] $Variable[[B]];
   ::$Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[BB]];
+)cpp",
+R"cpp(
+  struct $Class[[D]] {
+double $Field[[C]];
+  };
+  struct $Class[[A]] {
+double $Field[[B]];
+$Class[[D]] $Field[[E]];
+static double $Variable[[S]];
+void $Method[[foo]]() {
+  $Field[[B]] = 123;
+  this->$Field[[B]] = 156;
+  this->$Method[[foo]]();
+  $Method[[foo]]();
+  $Variable[[S]] = 90.1

[PATCH] D64634: [clangd] Fix duplicate highlighting tokens appearing in initializer lists

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom marked an inline comment as done.
jvikstrom added inline comments.



Comment at: clang-tools-extra/clangd/SemanticHighlighting.cpp:36
+// must be deduplicated.
+std::sort(Tokens.begin(), Tokens.end());
+for (unsigned I = 1; I < Tokens.size(); ++I) {

hokein wrote:
> nit: we could write it like
> 
> ```
> llvm::sort(Tokens, [](const HighlightingToken &Lhs, const HighlightingToken 
> &Rhs) {
>return std::tie(Lhs.Kind, Lhs.R) < std::tie(Rhs.Kind, Rhs.R);
> });
> auto Last = std::unique(Tokens.begin(), Tokens.end());
> Tokens.erase(Last, Tokens.end());
> ```
About the comparator function though. I added an < operator because 
https://reviews.llvm.org/D64475 also sorts the tokens. However, could I just 
remove the sort in the main diffHighlightings method and just rely on the 
highlightings to be sorted in the first place from this patch?

Instead we'd do the sorting like:
```
llvm::sort(Tokens, [](const HighlightingToken &Lhs, const HighlightingToken 
&Rhs) {
   return std::tie(Lhs.R,Lhs.Kind) < std::tie(Rhs.R,Rhs.Kind);
});
```

so we get it sorted by their position first and Kind second.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64634/new/

https://reviews.llvm.org/D64634



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64634: [clangd] Fix duplicate highlighting tokens appearing in initializer lists

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 209773.
jvikstrom added a comment.

Remove operator< and use std::unique.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64634/new/

https://reviews.llvm.org/D64634

Files:
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp


Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -166,6 +166,13 @@
 $Variable[[AA]].$Field[[E]].$Field[[C]];
 $Class[[A]]::$Variable[[S]] = 90;
   }
+)cpp",
+R"cpp(
+  struct $Class[[AA]] {
+int $Field[[A]];
+  }
+  int $Variable[[B]];
+  $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -31,6 +31,15 @@
   std::vector collectTokens() {
 Tokens.clear();
 TraverseAST(Ctx);
+// Initializer lists can give duplicates of tokens, therefore all tokens
+// must be deduplicated.
+llvm::sort(
+Tokens,
+[](const HighlightingToken &Lhs, const HighlightingToken &Rhs) -> bool 
{
+  return std::tie(Lhs.R, Lhs.Kind) < std::tie(Rhs.R, Rhs.Kind);
+});
+auto Last = std::unique(Tokens.begin(), Tokens.end());
+Tokens.erase(Last, Tokens.end());
 return Tokens;
   }
 
@@ -80,7 +89,6 @@
 DeclarationName::Identifier)
   // Only want to highlight identifiers.
   return true;
-
 addToken(Ref->getLocation(), Ref->getDecl());
 return true;
   }


Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -166,6 +166,13 @@
 $Variable[[AA]].$Field[[E]].$Field[[C]];
 $Class[[A]]::$Variable[[S]] = 90;
   }
+)cpp",
+R"cpp(
+  struct $Class[[AA]] {
+int $Field[[A]];
+  }
+  int $Variable[[B]];
+  $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -31,6 +31,15 @@
   std::vector collectTokens() {
 Tokens.clear();
 TraverseAST(Ctx);
+// Initializer lists can give duplicates of tokens, therefore all tokens
+// must be deduplicated.
+llvm::sort(
+Tokens,
+[](const HighlightingToken &Lhs, const HighlightingToken &Rhs) -> bool {
+  return std::tie(Lhs.R, Lhs.Kind) < std::tie(Rhs.R, Rhs.Kind);
+});
+auto Last = std::unique(Tokens.begin(), Tokens.end());
+Tokens.erase(Last, Tokens.end());
 return Tokens;
   }
 
@@ -80,7 +89,6 @@
 DeclarationName::Identifier)
   // Only want to highlight identifiers.
   return true;
-
 addToken(Ref->getLocation(), Ref->getDecl());
 return true;
   }
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64634: [clangd] Fix duplicate highlighting tokens appearing in initializer lists

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 209774.
jvikstrom added a comment.

Readded newline that was removed accidentaly.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64634/new/

https://reviews.llvm.org/D64634

Files:
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp


Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -166,6 +166,13 @@
 $Variable[[AA]].$Field[[E]].$Field[[C]];
 $Class[[A]]::$Variable[[S]] = 90;
   }
+)cpp",
+R"cpp(
+  struct $Class[[AA]] {
+int $Field[[A]];
+  }
+  int $Variable[[B]];
+  $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -31,6 +31,15 @@
   std::vector collectTokens() {
 Tokens.clear();
 TraverseAST(Ctx);
+// Initializer lists can give duplicates of tokens, therefore all tokens
+// must be deduplicated.
+llvm::sort(
+Tokens,
+[](const HighlightingToken &Lhs, const HighlightingToken &Rhs) -> bool 
{
+  return std::tie(Lhs.R, Lhs.Kind) < std::tie(Rhs.R, Rhs.Kind);
+});
+auto Last = std::unique(Tokens.begin(), Tokens.end());
+Tokens.erase(Last, Tokens.end());
 return Tokens;
   }
 


Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -166,6 +166,13 @@
 $Variable[[AA]].$Field[[E]].$Field[[C]];
 $Class[[A]]::$Variable[[S]] = 90;
   }
+)cpp",
+R"cpp(
+  struct $Class[[AA]] {
+int $Field[[A]];
+  }
+  int $Variable[[B]];
+  $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -31,6 +31,15 @@
   std::vector collectTokens() {
 Tokens.clear();
 TraverseAST(Ctx);
+// Initializer lists can give duplicates of tokens, therefore all tokens
+// must be deduplicated.
+llvm::sort(
+Tokens,
+[](const HighlightingToken &Lhs, const HighlightingToken &Rhs) -> bool {
+  return std::tie(Lhs.R, Lhs.Kind) < std::tie(Rhs.R, Rhs.Kind);
+});
+auto Last = std::unique(Tokens.begin(), Tokens.end());
+Tokens.erase(Last, Tokens.end());
 return Tokens;
   }
 
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64634: [clangd] Fix duplicate highlighting tokens appearing in initializer lists

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 209775.
jvikstrom added a comment.

Removed return type hint for lambda.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64634/new/

https://reviews.llvm.org/D64634

Files:
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp


Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -166,6 +166,13 @@
 $Variable[[AA]].$Field[[E]].$Field[[C]];
 $Class[[A]]::$Variable[[S]] = 90;
   }
+)cpp",
+R"cpp(
+  struct $Class[[AA]] {
+int $Field[[A]];
+  }
+  int $Variable[[B]];
+  $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -31,6 +31,14 @@
   std::vector collectTokens() {
 Tokens.clear();
 TraverseAST(Ctx);
+// Initializer lists can give duplicates of tokens, therefore all tokens
+// must be deduplicated.
+llvm::sort(Tokens,
+   [](const HighlightingToken &Lhs, const HighlightingToken &Rhs) {
+ return std::tie(Lhs.R, Lhs.Kind) < std::tie(Rhs.R, Rhs.Kind);
+   });
+auto Last = std::unique(Tokens.begin(), Tokens.end());
+Tokens.erase(Last, Tokens.end());
 return Tokens;
   }
 


Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -166,6 +166,13 @@
 $Variable[[AA]].$Field[[E]].$Field[[C]];
 $Class[[A]]::$Variable[[S]] = 90;
   }
+)cpp",
+R"cpp(
+  struct $Class[[AA]] {
+int $Field[[A]];
+  }
+  int $Variable[[B]];
+  $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -31,6 +31,14 @@
   std::vector collectTokens() {
 Tokens.clear();
 TraverseAST(Ctx);
+// Initializer lists can give duplicates of tokens, therefore all tokens
+// must be deduplicated.
+llvm::sort(Tokens,
+   [](const HighlightingToken &Lhs, const HighlightingToken &Rhs) {
+ return std::tie(Lhs.R, Lhs.Kind) < std::tie(Rhs.R, Rhs.Kind);
+   });
+auto Last = std::unique(Tokens.begin(), Tokens.end());
+Tokens.erase(Last, Tokens.end());
 return Tokens;
   }
 
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64741: [clangd] Added highlighting for tokens that are macro arguments.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom created this revision.
jvikstrom added reviewers: hokein, sammccall, ilya-biryukov.
Herald added subscribers: cfe-commits, kadircet, arphaman, jkorous, MaskRay.
Herald added a project: clang.

Adds semantic highlighting for tokens that are a macro argument.
Example:

  #define D_V(X) int X = 1230
  D_V(SomeVar);

The "SomeVar" inside the macro is highlighted as a variable now.

Tokens that are in a macro body expansion are ignored in this patch for three 
reasons.

- The spelling loc is inside the macro "definition" meaning it would highlight 
inside the macro definition (could probably easily be fixed by using 
getExpansionLoc instead of getSpellingLoc?)
- If wanting to highlight the macro definition this could create duplicate 
tokens. And if the tokens are of different types there would be conflicts 
(tokens in the same range but with different types). Say a macro defines some 
name and both a variable declaration and a function use this, there would be 
two tokens in the macro definition but one with Kind "Variable" and the other 
with Kind "Function".
- Thirdly, macro body expansions could come from a file that is not the main 
file (easily fixed, just check that the Loc is in the main file and not even a 
problem if we wanted to highlight the actual macro "invocation")


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D64741

Files:
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp


Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -166,6 +166,42 @@
 $Variable[[AA]].$Field[[E]].$Field[[C]];
 $Class[[A]]::$Variable[[S]] = 90;
   }
+)cpp",
+R"cpp(
+  #define DEF_CLASS(T) class T {};
+  DEF_CLASS($Class[[A]])
+
+  #define MACRO_CONCAT(X, V, T) T foo##X = V
+  #define DEF_VAR(X, V) int X = V
+  #define DEF_VAR_T(T, X, V) T X = V
+  #define DEF_VAR_REV(V, X) DEF_VAR(X, V)
+  #define CPY(X) X
+  #define DEF_VAR_TYPE(X, Y) X Y
+  #define SOME_NAME variable
+  #define SOME_NAME_SET variable2 = 123
+  #define INC_VAR(X) X += 2
+  void $Function[[foo]]() {
+DEF_VAR($Variable[[X]],  123);
+DEF_VAR_REV(908, $Variable[[XY]]);
+int CPY( $Variable[[XX]] );
+DEF_VAR_TYPE($Class[[A]], $Variable[[AA]]);
+double SOME_NAME;
+int SOME_NAME_SET;
+$Variable[[variable]] = 20.1;
+MACRO_CONCAT(var, 2, float);
+DEF_VAR_T($Class[[A]], CPY(CPY($Variable[[Nested]])),
+  CPY($Class[[A]]()));
+INC_VAR($Variable[[variable]]);
+  }
+  void SOME_NAME();
+  DEF_VAR($Variable[[XYZ]], 567);
+  DEF_VAR_REV(756, $Variable[[AB]]);
+
+  #define CALL_FN(F) F();
+  #define DEF_FN(F) void F ()
+  DEF_FN($Function[[g]]) {
+CALL_FN($Function[[foo]]);
+  }
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -160,9 +160,15 @@
   }
 
   void addToken(SourceLocation Loc, HighlightingKind Kind) {
-if (Loc.isMacroID())
-  // FIXME: skip tokens inside macros for now.
-  return;
+if(Loc.isMacroID()) {
+  // If the location is not an argument it might be from a macro of the 
form
+  // "#define VAR var". In that case this would highlight "var" in the 
macro
+  // definition and if VAR is used for functions and variables there would
+  // be conflicts and duplicates. So skip those for now.
+  if (!SM.isMacroArgExpansion(Loc))
+return;
+  Loc = SM.getSpellingLoc(Loc);
+}
 
 auto R = getTokenRange(SM, Ctx.getLangOpts(), Loc);
 if (!R) {


Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -166,6 +166,42 @@
 $Variable[[AA]].$Field[[E]].$Field[[C]];
 $Class[[A]]::$Variable[[S]] = 90;
   }
+)cpp",
+R"cpp(
+  #define DEF_CLASS(T) class T {};
+  DEF_CLASS($Class[[A]])
+
+  #define MACRO_CONCAT(X, V, T) T foo##X = V
+  #define DEF_VAR(X, V) int X = V
+  #define DEF_VAR_T(T, X, V) T X = V
+  #define DEF_VAR_REV(V, X) DEF_VAR(X, V)
+  #define CPY(X) X
+  #define DEF_VAR_TYPE(X, Y) X Y
+  #define SOME_NAME variable
+  #define SOME_NAME_SET variable2 = 123
+  #define INC_VAR(X) X += 2
+  void $Function[[foo]]() {

[PATCH] D64634: [clangd] Fix duplicate highlighting tokens appearing in initializer lists

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 209852.
jvikstrom marked an inline comment as done.
jvikstrom added a comment.

Address comment.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64634/new/

https://reviews.llvm.org/D64634

Files:
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp


Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -166,6 +166,13 @@
 $Variable[[AA]].$Field[[E]].$Field[[C]];
 $Class[[A]]::$Variable[[S]] = 90;
   }
+)cpp",
+R"cpp(
+  struct $Class[[AA]] {
+int $Field[[A]];
+  }
+  int $Variable[[B]];
+  $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -31,6 +31,14 @@
   std::vector collectTokens() {
 Tokens.clear();
 TraverseAST(Ctx);
+// Initializer lists can give duplicates of tokens, therefore all tokens
+// must be deduplicated.
+llvm::sort(Tokens,
+   [](const HighlightingToken &L, const HighlightingToken &R) {
+ return std::tie(L.R, L.Kind) < std::tie(R.R, R.Kind);
+   });
+auto Last = std::unique(Tokens.begin(), Tokens.end());
+Tokens.erase(Last, Tokens.end());
 return Tokens;
   }
 


Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -166,6 +166,13 @@
 $Variable[[AA]].$Field[[E]].$Field[[C]];
 $Class[[A]]::$Variable[[S]] = 90;
   }
+)cpp",
+R"cpp(
+  struct $Class[[AA]] {
+int $Field[[A]];
+  }
+  int $Variable[[B]];
+  $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -31,6 +31,14 @@
   std::vector collectTokens() {
 Tokens.clear();
 TraverseAST(Ctx);
+// Initializer lists can give duplicates of tokens, therefore all tokens
+// must be deduplicated.
+llvm::sort(Tokens,
+   [](const HighlightingToken &L, const HighlightingToken &R) {
+ return std::tie(L.R, L.Kind) < std::tie(R.R, R.Kind);
+   });
+auto Last = std::unique(Tokens.begin(), Tokens.end());
+Tokens.erase(Last, Tokens.end());
 return Tokens;
   }
 
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64634: [clangd] Fix duplicate highlighting tokens appearing in initializer lists

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom added a comment.

In D64634#1585521 , @ilya-biryukov 
wrote:

> Have we tried figuring out why `RecursiveASTVisitor` visits the argument 
> lists twice? Is that an expected behavior?


The comment for the function that traverses initialization lists says this:

> // This method is called once for each pair of syntactic and semantic
>  // InitListExpr, and it traverses the subtrees defined by the two forms. This
>  // may cause some of the children to be visited twice, if they appear both in
>  // the syntactic and the semantic form.

So it seems to be expected. (and looking at the documentation for InitListExpr 
it seems to be difficult to change RecursiveASTVisitor to visit every sub expr 
once)


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64634/new/

https://reviews.llvm.org/D64634



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64634: [clangd] Fix duplicate highlighting tokens appearing in initializer lists

[Johan Vikström via Phabricator via cfe-commits]

This revision was automatically updated to reflect the committed changes.
Closed by commit rL366070: [clangd] Fix duplicate highlighting tokens appearing 
in initializer lists. (authored by jvikstrom, committed by ).
Herald added a project: LLVM.
Herald added a subscriber: llvm-commits.

Changed prior to commit:
  https://reviews.llvm.org/D64634?vs=209852&id=209868#toc

Repository:
  rL LLVM

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64634/new/

https://reviews.llvm.org/D64634

Files:
  clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
  clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp


Index: clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
@@ -166,6 +166,13 @@
 $Variable[[AA]].$Field[[E]].$Field[[C]];
 $Class[[A]]::$Variable[[S]] = 90;
   }
+)cpp",
+R"cpp(
+  struct $Class[[AA]] {
+int $Field[[A]];
+  }
+  int $Variable[[B]];
+  $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
Index: clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
@@ -31,6 +31,14 @@
   std::vector collectTokens() {
 Tokens.clear();
 TraverseAST(Ctx);
+// Initializer lists can give duplicates of tokens, therefore all tokens
+// must be deduplicated.
+llvm::sort(Tokens,
+   [](const HighlightingToken &L, const HighlightingToken &R) {
+ return std::tie(L.R, L.Kind) < std::tie(R.R, R.Kind);
+   });
+auto Last = std::unique(Tokens.begin(), Tokens.end());
+Tokens.erase(Last, Tokens.end());
 return Tokens;
   }
 


Index: clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
@@ -166,6 +166,13 @@
 $Variable[[AA]].$Field[[E]].$Field[[C]];
 $Class[[A]]::$Variable[[S]] = 90;
   }
+)cpp",
+R"cpp(
+  struct $Class[[AA]] {
+int $Field[[A]];
+  }
+  int $Variable[[B]];
+  $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
Index: clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
@@ -31,6 +31,14 @@
   std::vector collectTokens() {
 Tokens.clear();
 TraverseAST(Ctx);
+// Initializer lists can give duplicates of tokens, therefore all tokens
+// must be deduplicated.
+llvm::sort(Tokens,
+   [](const HighlightingToken &L, const HighlightingToken &R) {
+ return std::tie(L.R, L.Kind) < std::tie(R.R, R.Kind);
+   });
+auto Last = std::unique(Tokens.begin(), Tokens.end());
+Tokens.erase(Last, Tokens.end());
 return Tokens;
   }
 
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64754: [clangd] Added highlighting for the targets in typedefs.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom created this revision.
jvikstrom added reviewers: hokein, sammccall, ilya-biryukov.
Herald added subscribers: cfe-commits, kadircet, arphaman, jkorous, MaskRay.
Herald added a project: clang.

In `typedef int A` the `A` was not highlighted previously.

This patch gives `A` the same kind of highlighting that the underlying type has 
(class/enum) (which in this example is no special highlighting because builtins 
are not handled yet)
Will add highlightings for built ins in another patch.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D64754

Files:
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp


Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -90,7 +90,7 @@
 typename T::A* $Field[[D]];
   };
   $Namespace[[abc]]::$Class[[A]] $Variable[[AA]];
-  typedef $Namespace[[abc]]::$Class[[A]] AAA;
+  typedef $Namespace[[abc]]::$Class[[A]] $Class[[AAA]];
   struct $Class[[B]] {
 $Class[[B]]();
 ~$Class[[B]]();
@@ -166,6 +166,16 @@
 $Variable[[AA]].$Field[[E]].$Field[[C]];
 $Class[[A]]::$Variable[[S]] = 90;
   }
+)cpp",
+R"cpp(
+  namespace $Namespace[[a]] {
+struct $Class[[A]] {};
+  }
+  typedef $Namespace[[a]]::$Class[[A]] $Class[[B]];
+  enum class $Enum[[E]] {};
+  typedef $Enum[[E]] $Enum[[C]];
+  typedef $Enum[[C]] $Enum[[CC]];
+  $Enum[[CC]] $Function[[f]]($Class[[B]]);
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -85,6 +85,12 @@
 return true;
   }
 
+  bool VisitTypedefDecl(TypedefDecl *TD) {
+if(const auto *TSI = TD->getTypeSourceInfo())
+  addTypeLoc(TD->getLocation(), TSI->getTypeLoc());
+return true;
+  }
+
   bool VisitTypeLoc(TypeLoc &TL) {
 // This check is for not getting two entries when there are anonymous
 // structs. It also makes us not highlight certain namespace qualifiers
@@ -93,9 +99,7 @@
 if (TL.getTypeLocClass() == TypeLoc::TypeLocClass::Elaborated)
   return true;
 
-if (const Type *TP = TL.getTypePtr())
-  if (const TagDecl *TD = TP->getAsTagDecl())
-addToken(TL.getBeginLoc(), TD);
+addTypeLoc(TL.getBeginLoc(), TL);
 return true;
   }
 
@@ -110,6 +114,12 @@
   }
 
 private:
+  void addTypeLoc(SourceLocation Loc, const TypeLoc &TL) {
+if (const Type *TP = TL.getTypePtr())
+  if (const TagDecl *TD = TP->getAsTagDecl())
+addToken(Loc, TD);
+  }
+
   void addToken(SourceLocation Loc, const NamedDecl *D) {
 if (D->getDeclName().isIdentifier() && D->getName().empty())
   // Don't add symbols that don't have any length.


Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -90,7 +90,7 @@
 typename T::A* $Field[[D]];
   };
   $Namespace[[abc]]::$Class[[A]] $Variable[[AA]];
-  typedef $Namespace[[abc]]::$Class[[A]] AAA;
+  typedef $Namespace[[abc]]::$Class[[A]] $Class[[AAA]];
   struct $Class[[B]] {
 $Class[[B]]();
 ~$Class[[B]]();
@@ -166,6 +166,16 @@
 $Variable[[AA]].$Field[[E]].$Field[[C]];
 $Class[[A]]::$Variable[[S]] = 90;
   }
+)cpp",
+R"cpp(
+  namespace $Namespace[[a]] {
+struct $Class[[A]] {};
+  }
+  typedef $Namespace[[a]]::$Class[[A]] $Class[[B]];
+  enum class $Enum[[E]] {};
+  typedef $Enum[[E]] $Enum[[C]];
+  typedef $Enum[[C]] $Enum[[CC]];
+  $Enum[[CC]] $Function[[f]]($Class[[B]]);
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -85,6 +85,12 @@
 return true;
   }
 
+  bool VisitTypedefDecl(TypedefDecl *TD) {
+if(const auto *TSI = TD->getTypeSourceInfo())
+  addTypeLoc(TD->getLocation(), TSI->getTypeLoc());
+return true;
+  }
+
   bool VisitTypeLoc(TypeLoc &TL) {
 // This check is for not getting two entries when there are anonymous
 // structs. It also makes us not highlight certain namespace qualifiers
@@ -93,9 +99,7 @@
 if (TL.getTypeLocClass() == TypeLoc::T

[PATCH] D64754: [clangd] Added highlighting for the targets in typedefs.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 210053.
jvikstrom added a comment.

Add support for using `A = ...`.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64754/new/

https://reviews.llvm.org/D64754

Files:
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp


Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -90,7 +90,7 @@
 typename T::A* $Field[[D]];
   };
   $Namespace[[abc]]::$Class[[A]] $Variable[[AA]];
-  typedef $Namespace[[abc]]::$Class[[A]] AAA;
+  typedef $Namespace[[abc]]::$Class[[A]] $Class[[AAA]];
   struct $Class[[B]] {
 $Class[[B]]();
 ~$Class[[B]]();
@@ -166,6 +166,19 @@
 $Variable[[AA]].$Field[[E]].$Field[[C]];
 $Class[[A]]::$Variable[[S]] = 90;
   }
+)cpp",
+R"cpp(
+  namespace $Namespace[[a]] {
+struct $Class[[A]] {};
+  }
+  typedef $Namespace[[a]]::$Class[[A]] $Class[[B]];
+  using $Class[[BB]] = $Namespace[[a]]::$Class[[A]];
+  enum class $Enum[[E]] {};
+  typedef $Enum[[E]] $Enum[[C]];
+  typedef $Enum[[C]] $Enum[[CC]];
+  using $Enum[[CD]] = $Enum[[CC]];
+  $Enum[[CC]] $Function[[f]]($Class[[B]]);
+  $Enum[[CD]] $Function[[f]]($Class[[BB]]);
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -85,6 +85,12 @@
 return true;
   }
 
+  bool VisitTypedefNameDecl(TypedefNameDecl *TD) {
+if(const auto *TSI = TD->getTypeSourceInfo())
+  addTypeLoc(TD->getLocation(), TSI->getTypeLoc());
+return true;
+  }
+
   bool VisitTypeLoc(TypeLoc &TL) {
 // This check is for not getting two entries when there are anonymous
 // structs. It also makes us not highlight certain namespace qualifiers
@@ -93,9 +99,7 @@
 if (TL.getTypeLocClass() == TypeLoc::TypeLocClass::Elaborated)
   return true;
 
-if (const Type *TP = TL.getTypePtr())
-  if (const TagDecl *TD = TP->getAsTagDecl())
-addToken(TL.getBeginLoc(), TD);
+addTypeLoc(TL.getBeginLoc(), TL);
 return true;
   }
 
@@ -110,6 +114,12 @@
   }
 
 private:
+  void addTypeLoc(SourceLocation Loc, const TypeLoc &TL) {
+if (const Type *TP = TL.getTypePtr())
+  if (const TagDecl *TD = TP->getAsTagDecl())
+addToken(Loc, TD);
+  }
+
   void addToken(SourceLocation Loc, const NamedDecl *D) {
 if (D->getDeclName().isIdentifier() && D->getName().empty())
   // Don't add symbols that don't have any length.


Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -90,7 +90,7 @@
 typename T::A* $Field[[D]];
   };
   $Namespace[[abc]]::$Class[[A]] $Variable[[AA]];
-  typedef $Namespace[[abc]]::$Class[[A]] AAA;
+  typedef $Namespace[[abc]]::$Class[[A]] $Class[[AAA]];
   struct $Class[[B]] {
 $Class[[B]]();
 ~$Class[[B]]();
@@ -166,6 +166,19 @@
 $Variable[[AA]].$Field[[E]].$Field[[C]];
 $Class[[A]]::$Variable[[S]] = 90;
   }
+)cpp",
+R"cpp(
+  namespace $Namespace[[a]] {
+struct $Class[[A]] {};
+  }
+  typedef $Namespace[[a]]::$Class[[A]] $Class[[B]];
+  using $Class[[BB]] = $Namespace[[a]]::$Class[[A]];
+  enum class $Enum[[E]] {};
+  typedef $Enum[[E]] $Enum[[C]];
+  typedef $Enum[[C]] $Enum[[CC]];
+  using $Enum[[CD]] = $Enum[[CC]];
+  $Enum[[CC]] $Function[[f]]($Class[[B]]);
+  $Enum[[CD]] $Function[[f]]($Class[[BB]]);
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -85,6 +85,12 @@
 return true;
   }
 
+  bool VisitTypedefNameDecl(TypedefNameDecl *TD) {
+if(const auto *TSI = TD->getTypeSourceInfo())
+  addTypeLoc(TD->getLocation(), TSI->getTypeLoc());
+return true;
+  }
+
   bool VisitTypeLoc(TypeLoc &TL) {
 // This check is for not getting two entries when there are anonymous
 // structs. It also makes us not highlight certain namespace qualifiers
@@ -93,9 +99,7 @@
 if (TL.getTypeLocClass() == TypeLoc::TypeLocClass::Ela

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 210079.
jvikstrom added a comment.

Moved highlighting state to LSP layer. Removed class holding state. Addressed 
comments.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475

Files:
  clang-tools-extra/clangd/ClangdLSPServer.cpp
  clang-tools-extra/clangd/ClangdLSPServer.h
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/semantic-highlighting.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,7 +29,9 @@
   return Tokens;
 }
 
-void checkHighlightings(llvm::StringRef Code) {
+std::tuple,
+   std::vector>
+getHighlightingsAnnotated(llvm::StringRef Code) {
   Annotations Test(Code);
   auto AST = TestTU::withCode(Test.code()).build();
   static const std::map KindToString{
@@ -49,9 +51,41 @@
   }
 
   auto ActualTokens = getSemanticHighlightings(AST);
+  return {std::move(AST), ActualTokens, ExpectedTokens};
+}
+
+void checkHighlightings(llvm::StringRef Code) {
+  std::vector ActualTokens;
+  std::vector ExpectedTokens;
+  std::tie(std::ignore, ActualTokens, ExpectedTokens) =
+  getHighlightingsAnnotated(Code);
   EXPECT_THAT(ActualTokens, testing::UnorderedElementsAreArray(ExpectedTokens));
 }
 
+void checkDiffedHighlights(const std::vector &ExpectedTokens,
+   const std::vector &EmptyLines,
+   std::vector &ActualDiffed) {
+  std::map> ExpectedLines;
+  for (const HighlightingToken &Token : ExpectedTokens)
+ExpectedLines[Token.R.start.line].push_back(Token);
+  std::vector ExpectedLinePairHighlighting;
+  for (int Line : EmptyLines)
+ExpectedLinePairHighlighting.push_back({Line, {}});
+  for (auto &LineTokens : ExpectedLines) {
+llvm::sort(LineTokens.second);
+ExpectedLinePairHighlighting.push_back(
+{LineTokens.first, LineTokens.second});
+  }
+
+  // The UnorderedElementsAreArray only checks that the top level vector
+  // is unordered. The vectors in the pair must be in the correct order.
+  for (unsigned I = 0, End = ActualDiffed.size(); I < End; ++I)
+llvm::sort(ActualDiffed[I].Tokens);
+
+  EXPECT_THAT(ActualDiffed,
+  testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting));
+}
+
 TEST(SemanticHighlighting, GetsCorrectTokens) {
   const char *TestCases[] = {
 R"cpp(
@@ -211,21 +245,103 @@
 return Pos;
   };
 
-  std::vector Tokens{
-  {HighlightingKind::Variable,
-Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
-  {HighlightingKind::Function,
-Range{CreatePosition(3, 4), CreatePosition(3, 7)}},
-  {HighlightingKind::Variable,
-Range{CreatePosition(1, 1), CreatePosition(1, 5)}}};
+  std::vector Tokens{
+  {3,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
+{HighlightingKind::Function,
+ Range{CreatePosition(3, 4), CreatePosition(3, 7),
+  {1,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(1, 1), CreatePosition(1, 5)};
   std::vector ActualResults =
   toSemanticHighlightingInformation(Tokens);
   std::vector ExpectedResults = {
-  {1, "AQAEAAA="},
-  {3, "CAAEAAAEAAMAAQ=="}};
+  {3, "CAAEAAAEAAMAAQ=="}, {1, "AQAEAAA="}};
   EXPECT_EQ(ActualResults, ExpectedResults);
 }
 
+TEST(SemanticHighlighting, HighlightingDiffer) {
+  std::vector<
+  std::pair, std::pair>>
+  TestCases{{{},
+ {
+ R"cpp(
+int $Variable[[A]]
+double $Variable[[B]];
+struct $Class[[C]] {};
+  )cpp",
+ R"cpp(
+int A;
+double B;
+struct C {};
+  )cpp"}},
+{{5},
+ {
+ R"cpp(
+  struct $Class[[Alpha]] {
+double SomeVariable = 9483.301;
+  };
+  struct $Class[[Beta]] {};
+  int $Variable[[A]] = 121;
+  $Class[[Alpha]] $Variable[[Var]];
+  )cpp",
+ R"cpp(
+  struct Alpha {
+double SomeVariable = 9483.301;
+  };
+  struct Beta   {}; // Some comment
+  intA = 121;
+  $Class[[Beta]] $Variable[[Var]];
+  )cpp"}},
+{{},
+ {
+ R"cpp(
+  int $Variable[[A]] = 121; int $Variable[[B]];
+)cpp",
+ R"cpp(
+  intA = 121; int $Variable[[B]];
+)cpp"}},
+{{},
+ {
+

[PATCH] D64754: [clangd] Added highlighting for the targets in typedefs.

[Johan Vikström via Phabricator via cfe-commits]

This revision was automatically updated to reflect the committed changes.
Closed by commit rL366207: [clangd] Added highlighting for the targets in 
typedefs and using. (authored by jvikstrom, committed by ).
Herald added a project: LLVM.
Herald added a subscriber: llvm-commits.

Changed prior to commit:
  https://reviews.llvm.org/D64754?vs=210053&id=210084#toc

Repository:
  rL LLVM

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64754/new/

https://reviews.llvm.org/D64754

Files:
  clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
  clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp


Index: clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
@@ -93,6 +93,12 @@
 return true;
   }
 
+  bool VisitTypedefNameDecl(TypedefNameDecl *TD) {
+if(const auto *TSI = TD->getTypeSourceInfo())
+  addTypeLoc(TD->getLocation(), TSI->getTypeLoc());
+return true;
+  }
+
   bool VisitTypeLoc(TypeLoc &TL) {
 // This check is for not getting two entries when there are anonymous
 // structs. It also makes us not highlight certain namespace qualifiers
@@ -101,9 +107,7 @@
 if (TL.getTypeLocClass() == TypeLoc::TypeLocClass::Elaborated)
   return true;
 
-if (const Type *TP = TL.getTypePtr())
-  if (const TagDecl *TD = TP->getAsTagDecl())
-addToken(TL.getBeginLoc(), TD);
+addTypeLoc(TL.getBeginLoc(), TL);
 return true;
   }
 
@@ -118,6 +122,12 @@
   }
 
 private:
+  void addTypeLoc(SourceLocation Loc, const TypeLoc &TL) {
+if (const Type *TP = TL.getTypePtr())
+  if (const TagDecl *TD = TP->getAsTagDecl())
+addToken(Loc, TD);
+  }
+
   void addToken(SourceLocation Loc, const NamedDecl *D) {
 if (D->getDeclName().isIdentifier() && D->getName().empty())
   // Don't add symbols that don't have any length.
Index: clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
@@ -90,7 +90,7 @@
 typename T::A* $Field[[D]];
   };
   $Namespace[[abc]]::$Class[[A]] $Variable[[AA]];
-  typedef $Namespace[[abc]]::$Class[[A]] AAA;
+  typedef $Namespace[[abc]]::$Class[[A]] $Class[[AAA]];
   struct $Class[[B]] {
 $Class[[B]]();
 ~$Class[[B]]();
@@ -173,6 +173,19 @@
   }
   int $Variable[[B]];
   $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
+)cpp",
+R"cpp(
+  namespace $Namespace[[a]] {
+struct $Class[[A]] {};
+  }
+  typedef $Namespace[[a]]::$Class[[A]] $Class[[B]];
+  using $Class[[BB]] = $Namespace[[a]]::$Class[[A]];
+  enum class $Enum[[E]] {};
+  typedef $Enum[[E]] $Enum[[C]];
+  typedef $Enum[[C]] $Enum[[CC]];
+  using $Enum[[CD]] = $Enum[[CC]];
+  $Enum[[CC]] $Function[[f]]($Class[[B]]);
+  $Enum[[CD]] $Function[[f]]($Class[[BB]]);
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);


Index: clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
@@ -93,6 +93,12 @@
 return true;
   }
 
+  bool VisitTypedefNameDecl(TypedefNameDecl *TD) {
+if(const auto *TSI = TD->getTypeSourceInfo())
+  addTypeLoc(TD->getLocation(), TSI->getTypeLoc());
+return true;
+  }
+
   bool VisitTypeLoc(TypeLoc &TL) {
 // This check is for not getting two entries when there are anonymous
 // structs. It also makes us not highlight certain namespace qualifiers
@@ -101,9 +107,7 @@
 if (TL.getTypeLocClass() == TypeLoc::TypeLocClass::Elaborated)
   return true;
 
-if (const Type *TP = TL.getTypePtr())
-  if (const TagDecl *TD = TP->getAsTagDecl())
-addToken(TL.getBeginLoc(), TD);
+addTypeLoc(TL.getBeginLoc(), TL);
 return true;
   }
 
@@ -118,6 +122,12 @@
   }
 
 private:
+  void addTypeLoc(SourceLocation Loc, const TypeLoc &TL) {
+if (const Type *TP = TL.getTypePtr())
+  if (const TagDecl *TD = TP->getAsTagDecl())
+addToken(Loc, TD);
+  }
+
   void addToken(SourceLocation Loc, const NamedDecl *D) {
 if (D->getDeclName().isIdentifier() && D->getName().empty())
   // Don't add symbols that don't have any length.
Index: clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
@@ -90,7 +90,7 @@
 typename 

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom added inline comments.



Comment at: clang-tools-extra/clangd/ClangdLSPServer.cpp:1110
 PathRef File, std::vector Highlightings) {
+  llvm::ArrayRef Prev;
+  {

hokein wrote:
> this seems unsafe, we get a reference of the map value, we might access it 
> without the mutex being guarded. 
> 
> ```
> std::vector Old;
> {
> std::lock_guard Lock(HighlightingsMutex);
> Old = std::move(FileToHighlightings[File]);
> }
> ```
Aren't the parsing callbacks thread safe in the same file? I think @sammccall 
said so above. 

Although actually, the map might reallocate and move around the memory if 
things are inserted/deleted invalidating the reference maybe (I have no idea of 
how StringMap memory allocation works).

So I guess it doesn't hurt to be safe and copy it.   



Comment at: clang-tools-extra/clangd/SemanticHighlighting.cpp:255
+  // ArrayRefs to the current line in the highlights.
+  ArrayRef NewLine(Highlightings.data(),
+  Highlightings.data()),

hokein wrote:
> IIUC, we are initializing an empty ArrayRef, if so, how about using 
> `NewLine(Highlightings.data(), /*length*/0)`? `NewLine(Highlightings.data(), 
> Highlightings.data())` is a bit weird, it took me a while to understand the 
> purpose.
I couldn't do that because the `ArrayRef(const T *data, size_t length)` and 
`ArrayRef(const T *begin, const T *end)` were ambiguous. Added a cast to cast 0 
to a size_t which solved it.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 210253.
jvikstrom marked 15 inline comments as done.
jvikstrom added a comment.

Address comments.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475

Files:
  clang-tools-extra/clangd/ClangdLSPServer.cpp
  clang-tools-extra/clangd/ClangdLSPServer.h
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/semantic-highlighting.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,7 +29,8 @@
   return Tokens;
 }
 
-void checkHighlightings(llvm::StringRef Code) {
+std::pair, std::vector>
+getHighlightingsAnnotated(llvm::StringRef Code) {
   Annotations Test(Code);
   auto AST = TestTU::withCode(Test.code()).build();
   static const std::map KindToString{
@@ -49,9 +50,40 @@
   }
 
   auto ActualTokens = getSemanticHighlightings(AST);
+  return {ActualTokens, ExpectedTokens};
+}
+
+void checkHighlightings(llvm::StringRef Code) {
+  std::vector ActualTokens;
+  std::vector ExpectedTokens;
+  std::tie(ActualTokens, ExpectedTokens) = getHighlightingsAnnotated(Code);
   EXPECT_THAT(ActualTokens, testing::UnorderedElementsAreArray(ExpectedTokens));
 }
 
+void checkDiffedHighlights(const std::vector &ExpectedTokens,
+   const std::vector &EmptyLines,
+   std::vector &ActualDiffed) {
+  std::map> ExpectedLines;
+  for (const HighlightingToken &Token : ExpectedTokens)
+ExpectedLines[Token.R.start.line].push_back(Token);
+  std::vector ExpectedLinePairHighlighting;
+  for (int Line : EmptyLines)
+ExpectedLinePairHighlighting.push_back({Line, {}});
+  for (auto &LineTokens : ExpectedLines) {
+llvm::sort(LineTokens.second);
+ExpectedLinePairHighlighting.push_back(
+{LineTokens.first, LineTokens.second});
+  }
+
+  // The UnorderedElementsAreArray only checks that the top level vector
+  // is unordered. The vectors in the pair must be in the correct order.
+  for (unsigned I = 0, End = ActualDiffed.size(); I < End; ++I)
+llvm::sort(ActualDiffed[I].Tokens);
+
+  EXPECT_THAT(ActualDiffed,
+  testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting));
+}
+
 TEST(SemanticHighlighting, GetsCorrectTokens) {
   const char *TestCases[] = {
 R"cpp(
@@ -211,21 +243,102 @@
 return Pos;
   };
 
-  std::vector Tokens{
-  {HighlightingKind::Variable,
-Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
-  {HighlightingKind::Function,
-Range{CreatePosition(3, 4), CreatePosition(3, 7)}},
-  {HighlightingKind::Variable,
-Range{CreatePosition(1, 1), CreatePosition(1, 5)}}};
+  std::vector Tokens{
+  {3,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
+{HighlightingKind::Function,
+ Range{CreatePosition(3, 4), CreatePosition(3, 7),
+  {1,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(1, 1), CreatePosition(1, 5)};
   std::vector ActualResults =
   toSemanticHighlightingInformation(Tokens);
   std::vector ExpectedResults = {
-  {1, "AQAEAAA="},
-  {3, "CAAEAAAEAAMAAQ=="}};
+  {3, "CAAEAAAEAAMAAQ=="}, {1, "AQAEAAA="}};
   EXPECT_EQ(ActualResults, ExpectedResults);
 }
 
+TEST(SemanticHighlighting, HighlightingDiffer) {
+  // Each entry is a test case. An entry consists of two things. The first entry
+  // in the pair is every empty line that should be returned by the diffing. The
+  // second entry are the two code samples that should be diffed (first entry
+  // are the old highlightings and the second entry are the new highlightings.)
+  std::vector<
+  std::pair, std::pair>>
+  TestCases{{{},
+ {
+ R"cpp(
+int $Variable[[A]]
+double $Variable[[B]];
+struct $Class[[C]] {};
+  )cpp",
+ R"cpp(
+int A;
+double B;
+struct C {};
+  )cpp"}},
+{{5},
+ {
+ R"cpp(
+  struct $Class[[Alpha]] {
+double SomeVariable = 9483.301;
+  };
+  struct $Class[[Beta]] {};
+  int $Variable[[A]] = 121;
+  $Class[[Alpha]] $Variable[[Var]];
+  )cpp",
+ R"cpp(
+  struct Alpha {
+double SomeVariable = 9483.301;
+  };
+  struct Beta   {}; // Some comment
+  intA = 121;
+  $Class[[Beta]] $Variable[[Var]];
+  )cpp"}},
+{{},
+ 

[PATCH] D64741: [clangd] Added highlighting for tokens that are macro arguments.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 210257.
jvikstrom marked 2 inline comments as done.
jvikstrom added a comment.

Address comments.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64741/new/

https://reviews.llvm.org/D64741

Files:
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,9 +29,11 @@
   return Tokens;
 }
 
-void checkHighlightings(llvm::StringRef Code) {
+void checkHighlightings(llvm::StringRef Code, llvm::StringRef HeaderCode = "") {
   Annotations Test(Code);
-  auto AST = TestTU::withCode(Test.code()).build();
+  TestTU TU = TestTU::withCode(Test.code());
+  TU.HeaderCode = HeaderCode;
+  auto AST = TU.build();
   static const std::map KindToString{
   {HighlightingKind::Variable, "Variable"},
   {HighlightingKind::Function, "Function"},
@@ -186,10 +188,57 @@
   using $Enum[[CD]] = $Enum[[CC]];
   $Enum[[CC]] $Function[[f]]($Class[[B]]);
   $Enum[[CD]] $Function[[f]]($Class[[BB]]);
+)cpp",
+R"cpp(
+  #define DEF_MULTIPLE(X) namespace X { class X { int X; }; }
+  #define DEF_CLASS(T) class T {};
+  DEF_CLASS($Class[[A]])
+  DEF_MULTIPLE(XYZ);
+
+  #define MACRO_CONCAT(X, V, T) T foo##X = V
+  #define DEF_VAR(X, V) int X = V
+  #define DEF_VAR_T(T, X, V) T X = V
+  #define DEF_VAR_REV(V, X) DEF_VAR(X, V)
+  #define CPY(X) X
+  #define DEF_VAR_TYPE(X, Y) X Y
+  #define SOME_NAME variable
+  #define SOME_NAME_SET variable2 = 123
+  #define INC_VAR(X) X += 2
+  void $Function[[foo]]() {
+DEF_VAR($Variable[[X]],  123);
+DEF_VAR_REV(908, $Variable[[XY]]);
+int CPY( $Variable[[XX]] );
+DEF_VAR_TYPE($Class[[A]], $Variable[[AA]]);
+double SOME_NAME;
+int SOME_NAME_SET;
+$Variable[[variable]] = 20.1;
+MACRO_CONCAT(var, 2, float);
+DEF_VAR_T($Class[[A]], CPY(CPY($Variable[[Nested]])),
+  CPY($Class[[A]]()));
+INC_VAR($Variable[[variable]]);
+  }
+  void SOME_NAME();
+  DEF_VAR($Variable[[XYZ]], 567);
+  DEF_VAR_REV(756, $Variable[[AB]]);
+
+  #define CALL_FN(F) F();
+  #define DEF_FN(F) void F ()
+  DEF_FN($Function[[g]]) {
+CALL_FN($Function[[foo]]);
+  }
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
   }
+
+  // A separate test for macros in headers.
+  checkHighlightings(R"cpp(
+#define DEFINE(X) int X;
+#define DEFINE_Y DEFINE(Y)
+  )cpp", R"cpp(
+DEFINE_Y
+DEFINE($Variable[[XYZ]]);
+  )cpp");
 }
 
 TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) {
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -12,6 +12,7 @@
 #include "SourceCode.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/RecursiveASTVisitor.h"
+#include "llvm/ADT/ArrayRef.h"
 
 namespace clang {
 namespace clangd {
@@ -39,6 +40,23 @@
});
 auto Last = std::unique(Tokens.begin(), Tokens.end());
 Tokens.erase(Last, Tokens.end());
+
+// Macros can give tokens that have the same source range but conflicting
+// kinds. In this case all tokens sharing this source range should be
+// removed.
+for (unsigned I = 0; I < Tokens.size(); ++I) {
+  ArrayRef TokRef(Tokens);
+  ArrayRef Conflicting =
+  llvm::ArrayRef(TokRef.begin() + I, TokRef.end())
+  .take_while([&](const HighlightingToken &T) {
+return T.R == Tokens[I].R;
+  });
+
+  if (Conflicting.size() > 1)
+Tokens.erase(Tokens.begin() + I,
+ Tokens.begin() + I + Conflicting.size());
+}
+
 return Tokens;
   }
 
@@ -178,9 +196,12 @@
   }
 
   void addToken(SourceLocation Loc, HighlightingKind Kind) {
-if (Loc.isMacroID())
-  // FIXME: skip tokens inside macros for now.
-  return;
+if(Loc.isMacroID()) {
+  // Only intereseted in highlighting arguments in macros (DEF_X(arg)).
+  if (!SM.isMacroArgExpansion(Loc))
+return;
+  Loc = SM.getSpellingLoc(Loc);
+}
 
 auto R = getTokenRange(SM, Ctx.getLangOpts(), Loc);
 if (!R) {
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64741: [clangd] Added highlighting for tokens that are macro arguments.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom marked an inline comment as done.
jvikstrom added inline comments.



Comment at: clang-tools-extra/clangd/SemanticHighlighting.cpp:170
+return;
+  Loc = SM.getSpellingLoc(Loc);
+}

hokein wrote:
> The Loc here maybe not in the main file, considering the case
> 
> ```
> // in .h
> #define DEFINE(X) in X;
> #define DEFINE_Y DEFINE(Y)
> 
> // in .cc
> 
> DEFINE_Y
> ```
The spelling loc is still going to be in `DEFINE_Y` I think. And we only 
highlight arguments in macros. (Added a testcase though)


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64741/new/

https://reviews.llvm.org/D64741



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64741: [clangd] Added highlighting for tokens that are macro arguments.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 210259.
jvikstrom marked an inline comment as done.
jvikstrom added a comment.

Fixed edge case when removing conflicting tokens.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64741/new/

https://reviews.llvm.org/D64741

Files:
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,9 +29,11 @@
   return Tokens;
 }
 
-void checkHighlightings(llvm::StringRef Code) {
+void checkHighlightings(llvm::StringRef Code, llvm::StringRef HeaderCode = "") {
   Annotations Test(Code);
-  auto AST = TestTU::withCode(Test.code()).build();
+  TestTU TU = TestTU::withCode(Test.code());
+  TU.HeaderCode = HeaderCode;
+  auto AST = TU.build();
   static const std::map KindToString{
   {HighlightingKind::Variable, "Variable"},
   {HighlightingKind::Function, "Function"},
@@ -186,10 +188,57 @@
   using $Enum[[CD]] = $Enum[[CC]];
   $Enum[[CC]] $Function[[f]]($Class[[B]]);
   $Enum[[CD]] $Function[[f]]($Class[[BB]]);
+)cpp",
+R"cpp(
+  #define DEF_MULTIPLE(X) namespace X { class X { int X; }; }
+  #define DEF_CLASS(T) class T {};
+  DEF_MULTIPLE(XYZ);
+  DEF_MULTIPLE(XYZW);
+  DEF_CLASS($Class[[A]])
+  #define MACRO_CONCAT(X, V, T) T foo##X = V
+  #define DEF_VAR(X, V) int X = V
+  #define DEF_VAR_T(T, X, V) T X = V
+  #define DEF_VAR_REV(V, X) DEF_VAR(X, V)
+  #define CPY(X) X
+  #define DEF_VAR_TYPE(X, Y) X Y
+  #define SOME_NAME variable
+  #define SOME_NAME_SET variable2 = 123
+  #define INC_VAR(X) X += 2
+  void $Function[[foo]]() {
+DEF_VAR($Variable[[X]],  123);
+DEF_VAR_REV(908, $Variable[[XY]]);
+int CPY( $Variable[[XX]] );
+DEF_VAR_TYPE($Class[[A]], $Variable[[AA]]);
+double SOME_NAME;
+int SOME_NAME_SET;
+$Variable[[variable]] = 20.1;
+MACRO_CONCAT(var, 2, float);
+DEF_VAR_T($Class[[A]], CPY(CPY($Variable[[Nested]])),
+  CPY($Class[[A]]()));
+INC_VAR($Variable[[variable]]);
+  }
+  void SOME_NAME();
+  DEF_VAR($Variable[[XYZ]], 567);
+  DEF_VAR_REV(756, $Variable[[AB]]);
+
+  #define CALL_FN(F) F();
+  #define DEF_FN(F) void F ()
+  DEF_FN($Function[[g]]) {
+CALL_FN($Function[[foo]]);
+  }
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
   }
+
+  // A separate test for macros in headers.
+  checkHighlightings(R"cpp(
+#define DEFINE(X) int X;
+#define DEFINE_Y DEFINE(Y)
+  )cpp", R"cpp(
+DEFINE_Y
+DEFINE($Variable[[XYZ]]);
+  )cpp");
 }
 
 TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) {
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -39,6 +39,26 @@
});
 auto Last = std::unique(Tokens.begin(), Tokens.end());
 Tokens.erase(Last, Tokens.end());
+
+// Macros can give tokens that have the same source range but conflicting
+// kinds. In this case all tokens sharing this source range should be
+// removed.
+for (unsigned I = 0; I < Tokens.size(); ++I) {
+  ArrayRef TokRef(Tokens);
+  ArrayRef Conflicting =
+  llvm::ArrayRef(TokRef.begin() + I, TokRef.end())
+  .take_while([&](const HighlightingToken &T) {
+return T.R == Tokens[I].R;
+  });
+
+  if (Conflicting.size() > 1) {
+Tokens.erase(Tokens.begin() + I,
+ Tokens.begin() + I + Conflicting.size());
+--I;
+  }
+
+}
+
 return Tokens;
   }
 
@@ -178,9 +198,12 @@
   }
 
   void addToken(SourceLocation Loc, HighlightingKind Kind) {
-if (Loc.isMacroID())
-  // FIXME: skip tokens inside macros for now.
-  return;
+if(Loc.isMacroID()) {
+  // Only intereseted in highlighting arguments in macros (DEF_X(arg)).
+  if (!SM.isMacroArgExpansion(Loc))
+return;
+  Loc = SM.getSpellingLoc(Loc);
+}
 
 auto R = getTokenRange(SM, Ctx.getLangOpts(), Loc);
 if (!R) {
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64741: [clangd] Added highlighting for tokens that are macro arguments.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom added a comment.

In D64741#1587204 , @ilya-biryukov 
wrote:

> How should this behave when if the same token is used multiple times inside a 
> macro and the uses are different? Could we add this to tests?
>  E.g.
>
>   #define W(a) class a { void test() { int a = 10; } }
>   W(foo); // <-- `foo` is a variable of a class?
>


I had completely missed that there could be conflicting tokens when only 
highlighting macro arguments as well. Added code to just remove conflicting 
tokens.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64741/new/

https://reviews.llvm.org/D64741



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64741: [clangd] Added highlighting for tokens that are macro arguments.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 210261.
jvikstrom added a comment.

clang-format


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64741/new/

https://reviews.llvm.org/D64741

Files:
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,9 +29,11 @@
   return Tokens;
 }
 
-void checkHighlightings(llvm::StringRef Code) {
+void checkHighlightings(llvm::StringRef Code, llvm::StringRef HeaderCode = "") {
   Annotations Test(Code);
-  auto AST = TestTU::withCode(Test.code()).build();
+  TestTU TU = TestTU::withCode(Test.code());
+  TU.HeaderCode = HeaderCode;
+  auto AST = TU.build();
   static const std::map KindToString{
   {HighlightingKind::Variable, "Variable"},
   {HighlightingKind::Function, "Function"},
@@ -186,10 +188,58 @@
   using $Enum[[CD]] = $Enum[[CC]];
   $Enum[[CC]] $Function[[f]]($Class[[B]]);
   $Enum[[CD]] $Function[[f]]($Class[[BB]]);
+)cpp",
+R"cpp(
+  #define DEF_MULTIPLE(X) namespace X { class X { int X; }; }
+  #define DEF_CLASS(T) class T {};
+  DEF_MULTIPLE(XYZ);
+  DEF_MULTIPLE(XYZW);
+  DEF_CLASS($Class[[A]])
+  #define MACRO_CONCAT(X, V, T) T foo##X = V
+  #define DEF_VAR(X, V) int X = V
+  #define DEF_VAR_T(T, X, V) T X = V
+  #define DEF_VAR_REV(V, X) DEF_VAR(X, V)
+  #define CPY(X) X
+  #define DEF_VAR_TYPE(X, Y) X Y
+  #define SOME_NAME variable
+  #define SOME_NAME_SET variable2 = 123
+  #define INC_VAR(X) X += 2
+  void $Function[[foo]]() {
+DEF_VAR($Variable[[X]],  123);
+DEF_VAR_REV(908, $Variable[[XY]]);
+int CPY( $Variable[[XX]] );
+DEF_VAR_TYPE($Class[[A]], $Variable[[AA]]);
+double SOME_NAME;
+int SOME_NAME_SET;
+$Variable[[variable]] = 20.1;
+MACRO_CONCAT(var, 2, float);
+DEF_VAR_T($Class[[A]], CPY(CPY($Variable[[Nested]])),
+  CPY($Class[[A]]()));
+INC_VAR($Variable[[variable]]);
+  }
+  void SOME_NAME();
+  DEF_VAR($Variable[[XYZ]], 567);
+  DEF_VAR_REV(756, $Variable[[AB]]);
+
+  #define CALL_FN(F) F();
+  #define DEF_FN(F) void F ()
+  DEF_FN($Function[[g]]) {
+CALL_FN($Function[[foo]]);
+  }
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
   }
+
+  // A separate test for macros in headers.
+  checkHighlightings(R"cpp(
+#define DEFINE(X) int X;
+#define DEFINE_Y DEFINE(Y)
+  )cpp",
+ R"cpp(
+DEFINE_Y
+DEFINE($Variable[[XYZ]]);
+  )cpp");
 }
 
 TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) {
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -39,6 +39,25 @@
});
 auto Last = std::unique(Tokens.begin(), Tokens.end());
 Tokens.erase(Last, Tokens.end());
+
+// Macros can give tokens that have the same source range but conflicting
+// kinds. In this case all tokens sharing this source range should be
+// removed.
+for (unsigned I = 0; I < Tokens.size(); ++I) {
+  ArrayRef TokRef(Tokens);
+  ArrayRef Conflicting =
+  llvm::ArrayRef(TokRef.begin() + I, TokRef.end())
+  .take_while([&](const HighlightingToken &T) {
+return T.R == Tokens[I].R;
+  });
+
+  if (Conflicting.size() > 1) {
+Tokens.erase(Tokens.begin() + I,
+ Tokens.begin() + I + Conflicting.size());
+--I;
+  }
+}
+
 return Tokens;
   }
 
@@ -178,9 +197,12 @@
   }
 
   void addToken(SourceLocation Loc, HighlightingKind Kind) {
-if (Loc.isMacroID())
-  // FIXME: skip tokens inside macros for now.
-  return;
+if(Loc.isMacroID()) {
+  // Only intereseted in highlighting arguments in macros (DEF_X(arg)).
+  if (!SM.isMacroArgExpansion(Loc))
+return;
+  Loc = SM.getSpellingLoc(Loc);
+}
 
 auto R = getTokenRange(SM, Ctx.getLangOpts(), Loc);
 if (!R) {
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64855: [clangd] Added highlightings for template parameters and specializations.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom created this revision.
jvikstrom added reviewers: hokein, sammccall, ilya-biryukov.
Herald added subscribers: cfe-commits, kadircet, arphaman, jkorous, MaskRay.
Herald added a project: clang.

Template parameters and specializations were not being highlighted before. This 
adds highlightings to those types of tokens by adding two Visit* methods.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D64855

Files:
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/semantic-highlighting.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -40,7 +40,8 @@
   {HighlightingKind::Namespace, "Namespace"},
   {HighlightingKind::EnumConstant, "EnumConstant"},
   {HighlightingKind::Field, "Field"},
-  {HighlightingKind::Method, "Method"}};
+  {HighlightingKind::Method, "Method"},
+  {HighlightingKind::TemplateParameter, "TemplateParameter"}};
   std::vector ExpectedTokens;
   for (const auto &KindString : KindToString) {
 std::vector Toks = makeHighlightingTokens(
@@ -80,14 +81,14 @@
 )cpp",
 R"cpp(
   namespace $Namespace[[abc]] {
-template
+template
 struct $Class[[A]] {
-  T $Field[[t]];
+  $TemplateParameter[[T]] $Field[[t]];
 };
   }
-  template
-  struct $Class[[C]] : $Namespace[[abc]]::A {
-typename T::A* $Field[[D]];
+  template
+  struct $Class[[C]] : $Namespace[[abc]]::$Class[[A]]<$TemplateParameter[[T]]> {
+typename $TemplateParameter[[T]]::A* $Field[[D]];
   };
   $Namespace[[abc]]::$Class[[A]] $Variable[[AA]];
   typedef $Namespace[[abc]]::$Class[[A]] AAA;
@@ -173,6 +174,19 @@
   }
   int $Variable[[B]];
   $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
+)cpp",
+R"cpp(
+  template
+  class $Class[[A]] {
+$TemplateParameter[[T]] $Field[[AA]];
+$TemplateParameter[[T]] $Method[[foo]]();
+  };
+  template
+  class $Class[[B]] {
+$Class[[A]]<$TemplateParameter[[TT]]> $Field[[AA]];
+  };
+  template
+  void $Function[[foo]]($TemplateParameter[[T]] ...);
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
Index: clang-tools-extra/clangd/test/semantic-highlighting.test
===
--- clang-tools-extra/clangd/test/semantic-highlighting.test
+++ clang-tools-extra/clangd/test/semantic-highlighting.test
@@ -27,6 +27,9 @@
 # CHECK-NEXT:  ],
 # CHECK-NEXT:  [
 # CHECK-NEXT:"entity.name.namespace.cpp"
+# CHECK-NEXT:  ],
+# CHECK-NEXT:  [
+# CHECK-NEXT:"entity.name.type.template.cpp"
 # CHECK-NEXT:  ]
 # CHECK-NEXT:]
 # CHECK-NEXT:  },
Index: clang-tools-extra/clangd/SemanticHighlighting.h
===
--- clang-tools-extra/clangd/SemanticHighlighting.h
+++ clang-tools-extra/clangd/SemanticHighlighting.h
@@ -32,6 +32,7 @@
   Enum,
   EnumConstant,
   Namespace,
+  TemplateParameter,
 
   NumKinds,
 };
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -93,6 +93,19 @@
 return true;
   }
 
+  bool VisitTemplateTypeParmTypeLoc(TemplateTypeParmTypeLoc &TL) {
+// TemplateTypeParmTypeLoc does not have a TagDecl in its type ptr.
+addToken(TL.getBeginLoc(), TL.getDecl());
+return true;
+  }
+
+  bool VisitTemplateSpecializationTypeLoc(TemplateSpecializationTypeLoc &TL) {
+if (const TemplateDecl *TD =
+TL.getTypePtr()->getTemplateName().getAsTemplateDecl())
+  addToken(TL.getBeginLoc(), TD);
+return true;
+  }
+
   bool VisitTypeLoc(TypeLoc &TL) {
 // This check is for not getting two entries when there are anonymous
 // structs. It also makes us not highlight certain namespace qualifiers
@@ -125,6 +138,10 @@
 // We highlight class decls, constructor decls and destructor decls as
 // `Class` type. The destructor decls are handled in `VisitTypeLoc` (we will
 // visit a TypeLoc where the underlying Type is a CXXRecordDecl).
+if(isa(D)) {
+  addToken(Loc, HighlightingKind::Class);
+  return;
+}
 if (isa(D)) {
   addToken(Loc, HighlightingKind::Class);
   return;
@@ -165,6 +182,10 @@
   addToken(Loc, HighlightingKind::Namespace);
   return;
 }
+if(isa(D)) {
+  addToken(Loc, HighlightingKind::TemplateParameter);
+ 

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 210282.
jvikstrom marked 4 inline comments as done.
jvikstrom added a comment.

Rewrote test to be more readable.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475

Files:
  clang-tools-extra/clangd/ClangdLSPServer.cpp
  clang-tools-extra/clangd/ClangdLSPServer.h
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/semantic-highlighting.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,7 +29,10 @@
   return Tokens;
 }
 
-void checkHighlightings(llvm::StringRef Code) {
+// Returns ActualTokens, ExpectedTokens and the lines that should be empty.
+std::tuple, std::vector,
+   std::vector>
+getHighlightingsAnnotated(llvm::StringRef Code) {
   Annotations Test(Code);
   auto AST = TestTU::withCode(Test.code()).build();
   static const std::map KindToString{
@@ -48,10 +51,56 @@
 ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
   }
 
+  auto EmptyRanges = Test.ranges("Empty");
+  std::vector EmptyLines(EmptyRanges.size());
+  for (unsigned I = 0, End = EmptyRanges.size(); I < End; ++I)
+EmptyLines[I] = EmptyRanges[I].start.line;
   auto ActualTokens = getSemanticHighlightings(AST);
+  return {ActualTokens, ExpectedTokens, EmptyLines};
+}
+
+void checkHighlightings(llvm::StringRef Code) {
+  std::vector ActualTokens;
+  std::vector ExpectedTokens;
+  std::tie(ActualTokens, ExpectedTokens, std::ignore) =
+  getHighlightingsAnnotated(Code);
   EXPECT_THAT(ActualTokens, testing::UnorderedElementsAreArray(ExpectedTokens));
 }
 
+void checkDiffedHighlights(const char *OrgCode, const char *NewCode) {
+  std::vector CompleteTokens1;
+  std::tie(CompleteTokens1, std::ignore, std::ignore) =
+  getHighlightingsAnnotated(OrgCode);
+  std::vector CompleteTokens2;
+  std::vector ExpectedTokens;
+  std::vector EmptyLines;
+  std::tie(CompleteTokens2, ExpectedTokens, EmptyLines) =
+  getHighlightingsAnnotated(NewCode);
+
+  std::vector ActualDiffed =
+  diffHighlightings(CompleteTokens2, CompleteTokens1);
+
+  std::map> ExpectedLines;
+  for (const HighlightingToken &Token : ExpectedTokens)
+ExpectedLines[Token.R.start.line].push_back(Token);
+  std::vector ExpectedLinePairHighlighting;
+  for (int Line : EmptyLines)
+ExpectedLinePairHighlighting.push_back({Line, {}});
+  for (auto &LineTokens : ExpectedLines) {
+llvm::sort(LineTokens.second);
+ExpectedLinePairHighlighting.push_back(
+{LineTokens.first, LineTokens.second});
+  }
+
+  // The UnorderedElementsAreArray only checks that the top level vector
+  // is unordered. The vectors in the pair must be in the correct order.
+  for (unsigned I = 0, End = ActualDiffed.size(); I < End; ++I)
+llvm::sort(ActualDiffed[I].Tokens);
+
+  EXPECT_THAT(ActualDiffed,
+  testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting));
+}
+
 TEST(SemanticHighlighting, GetsCorrectTokens) {
   const char *TestCases[] = {
 R"cpp(
@@ -211,21 +260,82 @@
 return Pos;
   };
 
-  std::vector Tokens{
-  {HighlightingKind::Variable,
-Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
-  {HighlightingKind::Function,
-Range{CreatePosition(3, 4), CreatePosition(3, 7)}},
-  {HighlightingKind::Variable,
-Range{CreatePosition(1, 1), CreatePosition(1, 5)}}};
+  std::vector Tokens{
+  {3,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
+{HighlightingKind::Function,
+ Range{CreatePosition(3, 4), CreatePosition(3, 7),
+  {1,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(1, 1), CreatePosition(1, 5)};
   std::vector ActualResults =
   toSemanticHighlightingInformation(Tokens);
   std::vector ExpectedResults = {
-  {1, "AQAEAAA="},
-  {3, "CAAEAAAEAAMAAQ=="}};
+  {3, "CAAEAAAEAAMAAQ=="}, {1, "AQAEAAA="}};
   EXPECT_EQ(ActualResults, ExpectedResults);
 }
 
+TEST(SemanticHighlighting, HighlightingDiffer) {
+  // The first entry is the old code. The second entry is the new code.
+  std::vector> TestCases{{
+   R"cpp(
+  int $Variable[[A]]
+  double $Variable[[B]];
+  struct $Class[[C]] {};
+)cpp",
+   R"cpp(
+  int A;
+  double B;
+  struct C {};
+)cpp"},
+   {

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom added inline comments.



Comment at: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp:281
+  )cpp"}},
+{{5},
+ {

hokein wrote:
> so the empty lines are stored separately, which is not easy to figure out 
> from the testing code snippet. I think we should make the empty lines more 
> obvious in the code snippet (maybe use an `Empty` annotations?) , and 
> explicitly verify the empty lines.
> 
> What do you think refining the test like below, just annotate the diff 
> tokens? I find it is easy to spot the diff tokens.
> 
> ```
> struct Testcase {
>Code Before;
>Code After;
> };
> 
> std::vector cases = {
>{ 
>R"cpp(
>  int a;
>  int b; 
>  int c;
>   )cpp",
>   R"cpp(
> int a;
> $Empty[[]] // int b
> int $Variable[[C]];
>   )cpp"
>   }
> }
> 
> oldHighlightings = getSemanticHighlightings(OldAST);
> newHighlightings = getSemanticHighlightings(NewAST);
> diffHighlightings = diff...;
> // verify the diffHighlightings has the expected empty lines ("Empty" 
> annotations).
> // verify the diffHighlightings has the expected highlightings (the regular 
> annotations);
> ```
> 
Moved everything into the checkDiffedHighlights as well.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64741: [clangd] Added highlighting for tokens that are macro arguments.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom added a comment.

In D64741#1588987 , @ilya-biryukov 
wrote:

> > I had completely missed that there could be conflicting tokens when only 
> > highlighting macro arguments as well. Added code to just remove conflicting 
> > tokens.
>
> Picking one of the highlightings looks fine, but we probably want to make 
> sure it's deterministic. Given that we use `sort` now, I bet it won't be. 
> Maybe include kinds into comparison as well? That's not a perfect solution, 
> but would at least make sure the user-visible behavior is not random.
>  Could you add tests for that case too?


Already added the case you sent a comment on in the test case. (look at the top 
of it)
Don't understand what you mean with the `sort` though. Kinds are already 
included in the comparator for sort. After the call to unique the only tokens 
that will share ranges are the ones that have different kinds.

Could just have a custom comparator in the call to unique that only compares 
the tokens' ranges which would leave us with the token whose kind is the lowest 
when there are conflicting ones. But do we really want to highlight anything 
when there are conflicts?

Maybe we should add another kind of Kind for when a token is conflicting?


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64741/new/

https://reviews.llvm.org/D64741



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 210297.
jvikstrom marked 6 inline comments as done.
jvikstrom added a comment.

Address comments.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475

Files:
  clang-tools-extra/clangd/ClangdLSPServer.cpp
  clang-tools-extra/clangd/ClangdLSPServer.h
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/semantic-highlighting.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,9 +29,12 @@
   return Tokens;
 }
 
-void checkHighlightings(llvm::StringRef Code) {
-  Annotations Test(Code);
+std::vector getActualTokens(Annotations &Test) {
   auto AST = TestTU::withCode(Test.code()).build();
+  return getSemanticHighlightings(AST);
+}
+
+std::vector getExpectedTokens(Annotations &Test) {
   static const std::map KindToString{
   {HighlightingKind::Variable, "Variable"},
   {HighlightingKind::Function, "Function"},
@@ -47,11 +50,50 @@
 Test.ranges(KindString.second), KindString.first);
 ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
   }
+  return ExpectedTokens;
+}
 
-  auto ActualTokens = getSemanticHighlightings(AST);
+std::vector getExpectedEmptyLines(Annotations &Test) {
+  auto EmptyRanges = Test.ranges("Empty");
+  std::vector EmptyLines(EmptyRanges.size());
+  for (unsigned I = 0, End = EmptyRanges.size(); I < End; ++I)
+EmptyLines[I] = EmptyRanges[I].start.line;
+  return EmptyLines;
+}
+
+void checkHighlightings(llvm::StringRef Code) {
+  Annotations Test(Code);
+  std::vector ActualTokens = getActualTokens(Test);
+  std::vector ExpectedTokens = getExpectedTokens(Test);
   EXPECT_THAT(ActualTokens, testing::UnorderedElementsAreArray(ExpectedTokens));
 }
 
+void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) {
+  Annotations OldTest(OldCode);
+  Annotations NewTest(NewCode);
+  std::vector OldActualTokens = getActualTokens(OldTest);
+  std::vector NewActualTokens = getActualTokens(NewTest);
+  std::vector ExpectedTokens = getExpectedTokens(NewTest);
+  std::vector EmptyLines = getExpectedEmptyLines(NewTest);
+  std::vector ActualDiffed =
+  diffHighlightings(NewActualTokens, OldActualTokens);
+
+  std::map> ExpectedLines;
+  for (const HighlightingToken &Token : ExpectedTokens)
+ExpectedLines[Token.R.start.line].push_back(Token);
+  std::vector ExpectedLinePairHighlighting;
+  for (int Line : EmptyLines)
+ExpectedLinePairHighlighting.push_back({Line, {}});
+  for (auto &LineTokens : ExpectedLines) {
+llvm::sort(LineTokens.second);
+ExpectedLinePairHighlighting.push_back(
+{LineTokens.first, LineTokens.second});
+  }
+
+  EXPECT_THAT(ActualDiffed,
+  testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting));
+}
+
 TEST(SemanticHighlighting, GetsCorrectTokens) {
   const char *TestCases[] = {
 R"cpp(
@@ -211,21 +253,82 @@
 return Pos;
   };
 
-  std::vector Tokens{
-  {HighlightingKind::Variable,
-Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
-  {HighlightingKind::Function,
-Range{CreatePosition(3, 4), CreatePosition(3, 7)}},
-  {HighlightingKind::Variable,
-Range{CreatePosition(1, 1), CreatePosition(1, 5)}}};
+  std::vector Tokens{
+  {3,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
+{HighlightingKind::Function,
+ Range{CreatePosition(3, 4), CreatePosition(3, 7),
+  {1,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(1, 1), CreatePosition(1, 5)};
   std::vector ActualResults =
   toSemanticHighlightingInformation(Tokens);
   std::vector ExpectedResults = {
-  {1, "AQAEAAA="},
-  {3, "CAAEAAAEAAMAAQ=="}};
+  {3, "CAAEAAAEAAMAAQ=="}, {1, "AQAEAAA="}};
   EXPECT_EQ(ActualResults, ExpectedResults);
 }
 
+TEST(SemanticHighlighting, HighlightingDiffer) {
+  // The first entry is the old code. The second entry is the new code.
+  std::vector> TestCases{{
+  R"cpp(
+int A
+double B;
+struct C {};
+  )cpp",
+  R"cpp(
+int A;
+double B;
+struct C {};
+  )cpp"},
+{
+  R"cpp(
+struct Alpha {
+  double SomeVariable = 9483.301;
+};
+struct Beta {};
+int A = 121;
+Alpha Var;
+  )cpp",
+  R"cpp(
+struct Alpha {
+  double SomeVariable = 9483.301;
+};
+struct Beta   {}; // Some comment
+$Empty[

[PATCH] D64741: [clangd] Added highlighting for tokens that are macro arguments.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 210359.
jvikstrom marked 2 inline comments as done.
jvikstrom added a comment.

Ignore tokens outside of main file. Added testcase for assert.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64741/new/

https://reviews.llvm.org/D64741

Files:
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,9 +29,11 @@
   return Tokens;
 }
 
-void checkHighlightings(llvm::StringRef Code) {
+void checkHighlightings(llvm::StringRef Code, llvm::StringRef HeaderCode = "") {
   Annotations Test(Code);
-  auto AST = TestTU::withCode(Test.code()).build();
+  TestTU TU = TestTU::withCode(Test.code());
+  TU.HeaderCode = HeaderCode;
+  auto AST = TU.build();
   static const std::map KindToString{
   {HighlightingKind::Variable, "Variable"},
   {HighlightingKind::Function, "Function"},
@@ -186,10 +188,70 @@
   using $Enum[[CD]] = $Enum[[CC]];
   $Enum[[CC]] $Function[[f]]($Class[[B]]);
   $Enum[[CD]] $Function[[f]]($Class[[BB]]);
+)cpp",
+R"cpp(
+  #define DEF_MULTIPLE(X) namespace X { class X { int X; }; }
+  #define DEF_CLASS(T) class T {};
+  DEF_MULTIPLE(XYZ);
+  DEF_MULTIPLE(XYZW);
+  DEF_CLASS($Class[[A]])
+  #define MACRO_CONCAT(X, V, T) T foo##X = V
+  #define DEF_VAR(X, V) int X = V
+  #define DEF_VAR_T(T, X, V) T X = V
+  #define DEF_VAR_REV(V, X) DEF_VAR(X, V)
+  #define CPY(X) X
+  #define DEF_VAR_TYPE(X, Y) X Y
+  #define SOME_NAME variable
+  #define SOME_NAME_SET variable2 = 123
+  #define INC_VAR(X) X += 2
+  void $Function[[foo]]() {
+DEF_VAR($Variable[[X]],  123);
+DEF_VAR_REV(908, $Variable[[XY]]);
+int CPY( $Variable[[XX]] );
+DEF_VAR_TYPE($Class[[A]], $Variable[[AA]]);
+double SOME_NAME;
+int SOME_NAME_SET;
+$Variable[[variable]] = 20.1;
+MACRO_CONCAT(var, 2, float);
+DEF_VAR_T($Class[[A]], CPY(CPY($Variable[[Nested]])),
+  CPY($Class[[A]]()));
+INC_VAR($Variable[[variable]]);
+  }
+  void SOME_NAME();
+  DEF_VAR($Variable[[XYZ]], 567);
+  DEF_VAR_REV(756, $Variable[[AB]]);
+
+  #define CALL_FN(F) F();
+  #define DEF_FN(F) void F ()
+  DEF_FN($Function[[g]]) {
+CALL_FN($Function[[foo]]);
+  }
+)cpp", R"cpp(
+  #define fail(expr) expr
+  #define assert(COND) if (!(COND)) { fail("assertion failed" #COND); }
+  int $Variable[[x]];
+  int $Variable[[y]];
+  int $Function[[f]]();
+  void $Function[[foo]]() {
+assert($Variable[[x]] != $Variable[[y]]);
+assert($Variable[[x]] != $Function[[f]]());
+  }
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
   }
+
+  // A separate test for macros in headers.
+  checkHighlightings(R"cpp(
+DEFINE_Y
+DXYZ_Y(A);
+  )cpp",
+ R"cpp(
+#define DXYZ(X) class X {};
+#define DXYZ_Y(Y) DXYZ(x##Y)
+#define DEFINE(X) in X;
+#define DEFINE_Y DEFINE(Y)
+  )cpp");
 }
 
 TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) {
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -39,6 +39,25 @@
});
 auto Last = std::unique(Tokens.begin(), Tokens.end());
 Tokens.erase(Last, Tokens.end());
+
+// Macros can give tokens that have the same source range but conflicting
+// kinds. In this case all tokens sharing this source range should be
+// removed.
+for (unsigned I = 0; I < Tokens.size(); ++I) {
+  ArrayRef TokRef(Tokens);
+  ArrayRef Conflicting =
+  llvm::ArrayRef(TokRef.begin() + I, TokRef.end())
+  .take_while([&](const HighlightingToken &T) {
+return T.R == Tokens[I].R;
+  });
+
+  if (Conflicting.size() > 1) {
+Tokens.erase(Tokens.begin() + I,
+ Tokens.begin() + I + Conflicting.size());
+--I;
+  }
+}
+
 return Tokens;
   }
 
@@ -178,9 +197,19 @@
   }
 
   void addToken(SourceLocation Loc, HighlightingKind Kind) {
-if (Loc.isMacroID())
-  // FIXME: skip tokens inside macros for now.
+if(Loc.isMacroID()) {
+  // Only intereseted in highlighting arguments in macros (DEF_X(arg)).
+  if (!SM.isMacroArgExpansion(Loc))
+return;
+  Loc = SM.getSpellingLoc(Loc);
+}
+
+if (!SM.isWrittenInMainFile(Loc)) {
+  // There are cases with macros where 

[PATCH] D64741: [clangd] Added highlighting for tokens that are macro arguments.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom added inline comments.



Comment at: clang-tools-extra/clangd/SemanticHighlighting.cpp:170
+return;
+  Loc = SM.getSpellingLoc(Loc);
+}

hokein wrote:
> hokein wrote:
> > jvikstrom wrote:
> > > hokein wrote:
> > > > The Loc here maybe not in the main file, considering the case
> > > > 
> > > > ```
> > > > // in .h
> > > > #define DEFINE(X) in X;
> > > > #define DEFINE_Y DEFINE(Y)
> > > > 
> > > > // in .cc
> > > > 
> > > > DEFINE_Y
> > > > ```
> > > The spelling loc is still going to be in `DEFINE_Y` I think. And we only 
> > > highlight arguments in macros. (Added a testcase though)
> > ok, I don't have an exact case for macros now, but my gut feeling tells me 
> > we will encounter cases where the Loc is not in main file.
> > 
> > here is a test case for declarations, we will visit the `foo()` decl in 
> > `test.h` as well. This could be addressed in a separate patch.
> > 
> > ```
> > // test.h
> > void foo();
> > 
> > // test.cc
> > #include "test.h"
> > void foo() {}
> > ```
> > 
> ok, here is the case, the spelling loc is not in main file, it is in 
> ``
> 
> ```
> // test.h
> #DEFINE(X) class X {};
> #DEFINE_Y(Y) DEFINE(x##Y)
> 
> // test.cc
> DEFINE_Y(a);
> ```
You are correct. The other comment was actually correct as well, I just put the 
arguments for the header code in the place where the source code should be in.

Added a check in addToken that the Loc we are trying to add must be in the main 
file.



Comment at: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp:195
+  #define DEF_CLASS(T) class T {};
+  DEF_MULTIPLE(XYZ);
+  DEF_MULTIPLE(XYZW);

ilya-biryukov wrote:
> Could you add a comment explaining that we choose to not highlight the 
> conflicting tokens?
There is a comment in SemanticHighlighting.cpp at line 43. Want me to add a 
comment in the test case as well?


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64741/new/

https://reviews.llvm.org/D64741



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64855: [clangd] Added highlightings for template parameters and specializations.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 210503.
jvikstrom added a comment.

Added testcases and made highlighting work for template template parameters as 
well.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64855/new/

https://reviews.llvm.org/D64855

Files:
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/semantic-highlighting.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -40,7 +40,8 @@
   {HighlightingKind::Namespace, "Namespace"},
   {HighlightingKind::EnumConstant, "EnumConstant"},
   {HighlightingKind::Field, "Field"},
-  {HighlightingKind::Method, "Method"}};
+  {HighlightingKind::Method, "Method"},
+  {HighlightingKind::TemplateParameter, "TemplateParameter"}};
   std::vector ExpectedTokens;
   for (const auto &KindString : KindToString) {
 std::vector Toks = makeHighlightingTokens(
@@ -80,14 +81,14 @@
 )cpp",
 R"cpp(
   namespace $Namespace[[abc]] {
-template
+template
 struct $Class[[A]] {
-  T $Field[[t]];
+  $TemplateParameter[[T]] $Field[[t]];
 };
   }
-  template
-  struct $Class[[C]] : $Namespace[[abc]]::A {
-typename T::A* $Field[[D]];
+  template
+  struct $Class[[C]] : $Namespace[[abc]]::$Class[[A]]<$TemplateParameter[[T]]> {
+typename $TemplateParameter[[T]]::A* $Field[[D]];
   };
   $Namespace[[abc]]::$Class[[A]] $Variable[[AA]];
   typedef $Namespace[[abc]]::$Class[[A]] AAA;
@@ -173,6 +174,29 @@
   }
   int $Variable[[B]];
   $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
+)cpp",
+R"cpp(
+  template
+  class $Class[[A]] {
+$TemplateParameter[[T]] $Field[[AA]];
+$TemplateParameter[[T]] $Method[[foo]]();
+  };
+  template
+  class $Class[[B]] {
+$Class[[A]]<$TemplateParameter[[TT]]> $Field[[AA]];
+  };
+  template
+  class $Class[[BB]] {};
+  template
+  class $Class[[BB]]<$TemplateParameter[[T]], int> {};
+  template
+  class $Class[[BB]]<$TemplateParameter[[T]], $TemplateParameter[[T]]*> {};
+
+  template class $TemplateParameter[[T]], class $TemplateParameter[[C]]>
+  $TemplateParameter[[T]]<$TemplateParameter[[C]]> $Function[[f]]();
+
+  template
+  void $Function[[foo]]($TemplateParameter[[T]] ...);
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
Index: clang-tools-extra/clangd/test/semantic-highlighting.test
===
--- clang-tools-extra/clangd/test/semantic-highlighting.test
+++ clang-tools-extra/clangd/test/semantic-highlighting.test
@@ -27,6 +27,9 @@
 # CHECK-NEXT:  ],
 # CHECK-NEXT:  [
 # CHECK-NEXT:"entity.name.namespace.cpp"
+# CHECK-NEXT:  ],
+# CHECK-NEXT:  [
+# CHECK-NEXT:"entity.name.type.template.cpp"
 # CHECK-NEXT:  ]
 # CHECK-NEXT:]
 # CHECK-NEXT:  },
Index: clang-tools-extra/clangd/SemanticHighlighting.h
===
--- clang-tools-extra/clangd/SemanticHighlighting.h
+++ clang-tools-extra/clangd/SemanticHighlighting.h
@@ -32,6 +32,7 @@
   Enum,
   EnumConstant,
   Namespace,
+  TemplateParameter,
 
   NumKinds,
 };
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -93,6 +93,19 @@
 return true;
   }
 
+  bool VisitTemplateTypeParmTypeLoc(TemplateTypeParmTypeLoc &TL) {
+// TemplateTypeParmTypeLoc does not have a TagDecl in its type ptr.
+addToken(TL.getBeginLoc(), TL.getDecl());
+return true;
+  }
+
+  bool VisitTemplateSpecializationTypeLoc(TemplateSpecializationTypeLoc &TL) {
+if (const TemplateDecl *TD =
+TL.getTypePtr()->getTemplateName().getAsTemplateDecl())
+  addToken(TL.getBeginLoc(), TD);
+return true;
+  }
+
   bool VisitTypeLoc(TypeLoc &TL) {
 // This check is for not getting two entries when there are anonymous
 // structs. It also makes us not highlight certain namespace qualifiers
@@ -125,6 +138,10 @@
 // We highlight class decls, constructor decls and destructor decls as
 // `Class` type. The destructor decls are handled in `VisitTypeLoc` (we will
 // visit a TypeLoc where the underlying Type is a CXXRecordDecl).
+if (isa(D)) {
+  addToken(Loc, HighlightingKind::Class);
+  return;
+}
 if (isa

[PATCH] D64855: [clangd] Added highlightings for template parameters and specializations.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom marked 2 inline comments as done.
jvikstrom added inline comments.



Comment at: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp:179
+R"cpp(
+  template
+  class $Class[[A]] {

hokein wrote:
> hokein wrote:
> > could you add some more test cases?
> > 
> > - for unnamed template parameter, `template  class Foo {};`
> > - for partial template specialization
> Looks like you are still missing the unnamed template parameter `template 
>  class Foo {};` case
Oh sorry, added it and landed.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64855/new/

https://reviews.llvm.org/D64855



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64855: [clangd] Added highlightings for template parameters and specializations.

[Johan Vikström via Phabricator via cfe-commits]

This revision was automatically updated to reflect the committed changes.
jvikstrom marked an inline comment as done.
Closed by commit rL366420: [clangd] Added highlightings for template parameters 
and specializations. (authored by jvikstrom, committed by ).
Herald added a project: LLVM.
Herald added a subscriber: llvm-commits.

Changed prior to commit:
  https://reviews.llvm.org/D64855?vs=210503&id=210510#toc

Repository:
  rL LLVM

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64855/new/

https://reviews.llvm.org/D64855

Files:
  clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
  clang-tools-extra/trunk/clangd/SemanticHighlighting.h
  clang-tools-extra/trunk/clangd/test/semantic-highlighting.test
  clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
@@ -99,6 +99,19 @@
 return true;
   }
 
+  bool VisitTemplateTypeParmTypeLoc(TemplateTypeParmTypeLoc &TL) {
+// TemplateTypeParmTypeLoc does not have a TagDecl in its type ptr.
+addToken(TL.getBeginLoc(), TL.getDecl());
+return true;
+  }
+
+  bool VisitTemplateSpecializationTypeLoc(TemplateSpecializationTypeLoc &TL) {
+if (const TemplateDecl *TD =
+TL.getTypePtr()->getTemplateName().getAsTemplateDecl())
+  addToken(TL.getBeginLoc(), TD);
+return true;
+  }
+
   bool VisitTypeLoc(TypeLoc &TL) {
 // This check is for not getting two entries when there are anonymous
 // structs. It also makes us not highlight certain namespace qualifiers
@@ -135,6 +148,10 @@
 // We highlight class decls, constructor decls and destructor decls as
 // `Class` type. The destructor decls are handled in `VisitTypeLoc` (we will
 // visit a TypeLoc where the underlying Type is a CXXRecordDecl).
+if (isa(D)) {
+  addToken(Loc, HighlightingKind::Class);
+  return;
+}
 if (isa(D)) {
   addToken(Loc, HighlightingKind::Class);
   return;
@@ -175,6 +192,14 @@
   addToken(Loc, HighlightingKind::Namespace);
   return;
 }
+if (isa(D)) {
+  addToken(Loc, HighlightingKind::TemplateParameter);
+  return;
+}
+if (isa(D)) {
+  addToken(Loc, HighlightingKind::TemplateParameter);
+  return;
+}
   }
 
   void addToken(SourceLocation Loc, HighlightingKind Kind) {
@@ -297,6 +322,8 @@
 return "variable.other.enummember.cpp";
   case HighlightingKind::Namespace:
 return "entity.name.namespace.cpp";
+  case HighlightingKind::TemplateParameter:
+return "entity.name.type.template.cpp";
   case HighlightingKind::NumKinds:
 llvm_unreachable("must not pass NumKinds to the function");
   }
Index: clang-tools-extra/trunk/clangd/SemanticHighlighting.h
===
--- clang-tools-extra/trunk/clangd/SemanticHighlighting.h
+++ clang-tools-extra/trunk/clangd/SemanticHighlighting.h
@@ -32,6 +32,7 @@
   Enum,
   EnumConstant,
   Namespace,
+  TemplateParameter,
 
   NumKinds,
 };
Index: clang-tools-extra/trunk/clangd/test/semantic-highlighting.test
===
--- clang-tools-extra/trunk/clangd/test/semantic-highlighting.test
+++ clang-tools-extra/trunk/clangd/test/semantic-highlighting.test
@@ -27,6 +27,9 @@
 # CHECK-NEXT:  ],
 # CHECK-NEXT:  [
 # CHECK-NEXT:"entity.name.namespace.cpp"
+# CHECK-NEXT:  ],
+# CHECK-NEXT:  [
+# CHECK-NEXT:"entity.name.type.template.cpp"
 # CHECK-NEXT:  ]
 # CHECK-NEXT:]
 # CHECK-NEXT:  },
Index: clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
@@ -40,7 +40,8 @@
   {HighlightingKind::Namespace, "Namespace"},
   {HighlightingKind::EnumConstant, "EnumConstant"},
   {HighlightingKind::Field, "Field"},
-  {HighlightingKind::Method, "Method"}};
+  {HighlightingKind::Method, "Method"},
+  {HighlightingKind::TemplateParameter, "TemplateParameter"}};
   std::vector ExpectedTokens;
   for (const auto &KindString : KindToString) {
 std::vector Toks = makeHighlightingTokens(
@@ -80,14 +81,14 @@
 )cpp",
 R"cpp(
   namespace $Namespace[[abc]] {
-template
+template
 struct $Class[[A]] {
-  T $Field[[t]];
+  $TemplateParameter[[T]] $Field[[t]];
 };
   }
-  template
-  struct $Class[[C]] : $Namespace[[abc]]::A {
-typename T::A* $Field[[D]];
+  template
+  struct $Class[[C]] : $Namespace[[abc]]::$Class[[A]]<$TemplateParameter[[T]]> {
+typ

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 210527.
jvikstrom marked 9 inline comments as done.
jvikstrom added a comment.

Address comments.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475

Files:
  clang-tools-extra/clangd/ClangdLSPServer.cpp
  clang-tools-extra/clangd/ClangdLSPServer.h
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/semantic-highlighting.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,9 +29,12 @@
   return Tokens;
 }
 
-void checkHighlightings(llvm::StringRef Code) {
-  Annotations Test(Code);
+std::vector getActualTokens(Annotations &Test) {
   auto AST = TestTU::withCode(Test.code()).build();
+  return getSemanticHighlightings(AST);
+}
+
+std::vector getExpectedTokens(Annotations &Test) {
   static const std::map KindToString{
   {HighlightingKind::Variable, "Variable"},
   {HighlightingKind::Function, "Function"},
@@ -48,9 +51,43 @@
 Test.ranges(KindString.second), KindString.first);
 ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
   }
+  llvm::sort(ExpectedTokens);
+  return ExpectedTokens;
+}
 
-  auto ActualTokens = getSemanticHighlightings(AST);
-  EXPECT_THAT(ActualTokens, testing::UnorderedElementsAreArray(ExpectedTokens));
+std::vector getExpectedEmptyLines(Annotations &Test) {
+  auto EmptyRanges = Test.ranges("Empty");
+  std::vector EmptyLines;
+  EmptyLines.reserve(EmptyRanges.size());
+  for (const auto &EmptyRange : EmptyRanges)
+EmptyLines.push_back(EmptyRange.start.line);
+  return EmptyLines;
+}
+
+void checkHighlightings(llvm::StringRef Code) {
+  Annotations Test(Code);
+  EXPECT_THAT(getActualTokens(Test),
+  getExpectedTokens(Test));
+}
+
+void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) {
+  Annotations OldTest(OldCode);
+  Annotations NewTest(NewCode);
+  std::vector ExpectedLinePairHighlighting;
+  for (int Line : getExpectedEmptyLines(NewTest))
+ExpectedLinePairHighlighting.push_back({Line, {}});
+  std::map> ExpectedLines;
+  for (const HighlightingToken &Token : getExpectedTokens(NewTest))
+ExpectedLines[Token.R.start.line].push_back(Token);
+  for (auto &LineTokens : ExpectedLines) {
+ExpectedLinePairHighlighting.push_back(
+{LineTokens.first, LineTokens.second});
+  }
+
+  std::vector ActualDiffed =
+  diffHighlightings(getActualTokens(NewTest), getActualTokens(OldTest));
+  EXPECT_THAT(ActualDiffed,
+  testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting));
 }
 
 TEST(SemanticHighlighting, GetsCorrectTokens) {
@@ -251,21 +288,110 @@
 return Pos;
   };
 
-  std::vector Tokens{
-  {HighlightingKind::Variable,
-Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
-  {HighlightingKind::Function,
-Range{CreatePosition(3, 4), CreatePosition(3, 7)}},
-  {HighlightingKind::Variable,
-Range{CreatePosition(1, 1), CreatePosition(1, 5)}}};
+  std::vector Tokens{
+  {3,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
+{HighlightingKind::Function,
+ Range{CreatePosition(3, 4), CreatePosition(3, 7),
+  {1,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(1, 1), CreatePosition(1, 5)};
   std::vector ActualResults =
   toSemanticHighlightingInformation(Tokens);
   std::vector ExpectedResults = {
-  {1, "AQAEAAA="},
-  {3, "CAAEAAAEAAMAAQ=="}};
+  {3, "CAAEAAAEAAMAAQ=="}, {1, "AQAEAAA="}};
   EXPECT_EQ(ActualResults, ExpectedResults);
 }
 
+TEST(SemanticHighlighting, HighlightingDiffer) {
+  struct {
+llvm::StringRef OldCode;
+llvm::StringRef NewCode;
+  } TestCases[]{{
+  R"cpp(
+int A
+double B;
+struct C {};
+  )cpp",
+  R"cpp(
+int A;
+double B;
+struct C {};
+  )cpp"},
+{
+  R"cpp(
+struct Alpha {
+  double SomeVariable = 9483.301;
+};
+struct Beta {};
+int A = 121;
+Alpha Var;
+  )cpp",
+  R"cpp(
+struct Alpha {
+  double SomeVariable = 9483.301;
+};
+struct Beta   {}; // Some comment
+$Variable[[intA]] = 121;
+$Class[[Beta]] $Variable[[Var]];
+  )cpp"},
+{
+  R"cpp(
+int A = 121; int B;
+  )cpp",
+  R"cpp(
+$Variable[[intA]] = 121; int $Variable[[B]];
+  )cpp"},
+{
+  R"cpp(
+intA;
+  )cp

[PATCH] D64741: [clangd] Added highlighting for tokens that are macro arguments.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom marked an inline comment as done.
jvikstrom added inline comments.



Comment at: clang-tools-extra/clangd/SemanticHighlighting.cpp:46
+// removed.
+for (unsigned I = 0; I < Tokens.size(); ++I) {
+  ArrayRef TokRef(Tokens);

hokein wrote:
> we don't care the Kind in `HighlightingToken` now, I think we could simplify 
> the code by tweaking the deduplication logic above?
> 
> ```
> llvm::sort(Tokens, [](const HighlightingToken &L, const HighlightingToken &R) 
> {
>  return L.R < R.R;
>});
> std::unique(Tokens.begin(), Tokens.end(), [](const HighlightingToken &L, 
> const HighlightingToken &R) {
>  return L.R == R.R;
>});
> ```
This would still keep one entry of the conflicting token though. (If we have 
one Kind that is a variable and one that is a function. One of those tokens 
would still be in Tokens at the place they were conflicting as unique removes 
every element but one of duplicates)

If we have conflicting tokens we want to remove all of them because it doesn't 
really make sense to highlight them as anything.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64741/new/

https://reviews.llvm.org/D64741



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64741: [clangd] Added highlighting for tokens that are macro arguments.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 210530.
jvikstrom added a comment.

Added comment saying conflicting tokens are not highlighted in the test.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64741/new/

https://reviews.llvm.org/D64741

Files:
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,9 +29,11 @@
   return Tokens;
 }
 
-void checkHighlightings(llvm::StringRef Code) {
+void checkHighlightings(llvm::StringRef Code, llvm::StringRef HeaderCode = "") {
   Annotations Test(Code);
-  auto AST = TestTU::withCode(Test.code()).build();
+  TestTU TU = TestTU::withCode(Test.code());
+  TU.HeaderCode = HeaderCode;
+  auto AST = TU.build();
   static const std::map KindToString{
   {HighlightingKind::Variable, "Variable"},
   {HighlightingKind::Function, "Function"},
@@ -186,10 +188,72 @@
   using $Enum[[CD]] = $Enum[[CC]];
   $Enum[[CC]] $Function[[f]]($Class[[B]]);
   $Enum[[CD]] $Function[[f]]($Class[[BB]]);
+)cpp",
+  // Tokens that share a source range but have conflicting Kinds are not
+  // highlighted.
+R"cpp(
+  #define DEF_MULTIPLE(X) namespace X { class X { int X; }; }
+  #define DEF_CLASS(T) class T {};
+  DEF_MULTIPLE(XYZ);
+  DEF_MULTIPLE(XYZW);
+  DEF_CLASS($Class[[A]])
+  #define MACRO_CONCAT(X, V, T) T foo##X = V
+  #define DEF_VAR(X, V) int X = V
+  #define DEF_VAR_T(T, X, V) T X = V
+  #define DEF_VAR_REV(V, X) DEF_VAR(X, V)
+  #define CPY(X) X
+  #define DEF_VAR_TYPE(X, Y) X Y
+  #define SOME_NAME variable
+  #define SOME_NAME_SET variable2 = 123
+  #define INC_VAR(X) X += 2
+  void $Function[[foo]]() {
+DEF_VAR($Variable[[X]],  123);
+DEF_VAR_REV(908, $Variable[[XY]]);
+int CPY( $Variable[[XX]] );
+DEF_VAR_TYPE($Class[[A]], $Variable[[AA]]);
+double SOME_NAME;
+int SOME_NAME_SET;
+$Variable[[variable]] = 20.1;
+MACRO_CONCAT(var, 2, float);
+DEF_VAR_T($Class[[A]], CPY(CPY($Variable[[Nested]])),
+  CPY($Class[[A]]()));
+INC_VAR($Variable[[variable]]);
+  }
+  void SOME_NAME();
+  DEF_VAR($Variable[[XYZ]], 567);
+  DEF_VAR_REV(756, $Variable[[AB]]);
+
+  #define CALL_FN(F) F();
+  #define DEF_FN(F) void F ()
+  DEF_FN($Function[[g]]) {
+CALL_FN($Function[[foo]]);
+  }
+)cpp", R"cpp(
+  #define fail(expr) expr
+  #define assert(COND) if (!(COND)) { fail("assertion failed" #COND); }
+  int $Variable[[x]];
+  int $Variable[[y]];
+  int $Function[[f]]();
+  void $Function[[foo]]() {
+assert($Variable[[x]] != $Variable[[y]]);
+assert($Variable[[x]] != $Function[[f]]());
+  }
 )cpp"};
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
   }
+
+  // A separate test for macros in headers.
+  checkHighlightings(R"cpp(
+DEFINE_Y
+DXYZ_Y(A);
+  )cpp",
+ R"cpp(
+#define DXYZ(X) class X {};
+#define DXYZ_Y(Y) DXYZ(x##Y)
+#define DEFINE(X) in X;
+#define DEFINE_Y DEFINE(Y)
+  )cpp");
 }
 
 TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) {
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -39,6 +39,25 @@
});
 auto Last = std::unique(Tokens.begin(), Tokens.end());
 Tokens.erase(Last, Tokens.end());
+
+// Macros can give tokens that have the same source range but conflicting
+// kinds. In this case all tokens sharing this source range should be
+// removed.
+for (unsigned I = 0; I < Tokens.size(); ++I) {
+  ArrayRef TokRef(Tokens);
+  ArrayRef Conflicting =
+  llvm::ArrayRef(TokRef.begin() + I, TokRef.end())
+  .take_while([&](const HighlightingToken &T) {
+return T.R == Tokens[I].R;
+  });
+
+  if (Conflicting.size() > 1) {
+Tokens.erase(Tokens.begin() + I,
+ Tokens.begin() + I + Conflicting.size());
+--I;
+  }
+}
+
 return Tokens;
   }
 
@@ -178,9 +197,19 @@
   }
 
   void addToken(SourceLocation Loc, HighlightingKind Kind) {
-if (Loc.isMacroID())
-  // FIXME: skip tokens inside macros for now.
+if(Loc.isMacroID()) {
+  // Only intereseted in highlighting arguments in macros (DEF_X(arg)).
+  if (!SM.isMacroArgExpansion(Loc))
+return;
+  Loc = SM.getSpellingLoc(Loc);
+}
+
+if (!SM.isWri

[PATCH] D64922: [clangd] Added option to enable semantic highlighting via an experimental capability

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom created this revision.
jvikstrom added reviewers: hokein, sammccall, ilya-biryukov.
Herald added subscribers: cfe-commits, kadircet, arphaman, jkorous, MaskRay.
Herald added a project: clang.

Adds option to enable semantic highlighting as an experimental capability. The 
reason for this is that it is not possible to add textDocument capabilities in 
the vscode extension as it is a strongly typed argument. The experimental 
capability is a normal JS like object and we can therefore send the capability 
there. However we can't remove the textDocument capability as theia relies on 
the capability to be in textDocument. (The same reasoning is why the server 
sends the TM scopes in the experimental capability if it got the capability as 
experimental).


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D64922

Files:
  clang-tools-extra/clangd/ClangdLSPServer.cpp
  clang-tools-extra/clangd/Protocol.cpp
  clang-tools-extra/clangd/Protocol.h
  clang-tools-extra/clangd/test/experimental-semantic-highlighting.test
  clang-tools-extra/clangd/test/semantic-highlighting.test

Index: clang-tools-extra/clangd/test/semantic-highlighting.test
===
--- clang-tools-extra/clangd/test/semantic-highlighting.test
+++ clang-tools-extra/clangd/test/semantic-highlighting.test
@@ -2,7 +2,7 @@
 {"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{"textDocument":{"semanticHighlightingCapabilities":{"semanticHighlighting":true}}},"trace":"off"}}
 ---
 #  CHECK:  "id": 0,
-# CHECK:  "semanticHighlighting": {
+#  CHECK:  "semanticHighlighting": {
 # CHECK-NEXT:"scopes": [
 # CHECK-NEXT:  [
 # CHECK-NEXT:"variable.other.cpp"
Index: clang-tools-extra/clangd/test/experimental-semantic-highlighting.test
===
--- /dev/null
+++ clang-tools-extra/clangd/test/experimental-semantic-highlighting.test
@@ -0,0 +1,41 @@
+# RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s
+{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{"experimental":{"semanticHighlightingCapabilities":{"semanticHighlighting":true}}},"trace":"off"}}
+---
+#  CHECK:  "id": 0,
+#  CHECK:  "experimental": {
+# CHECK-NEXT:"semanticHighlighting": {
+# CHECK-NEXT:  "scopes": [
+# CHECK-NEXT:[
+# CHECK-NEXT:  "variable.other.cpp"
+# CHECK-NEXT:],
+# CHECK-NEXT:[
+# CHECK-NEXT:  "entity.name.function.cpp"
+# CHECK-NEXT:],
+# CHECK-NEXT:[
+# CHECK-NEXT:  "entity.name.function.method.cpp"
+# CHECK-NEXT:],
+# CHECK-NEXT:[
+# CHECK-NEXT:  "variable.other.field.cpp"
+# CHECK-NEXT:],
+# CHECK-NEXT:[
+# CHECK-NEXT:  "entity.name.type.class.cpp"
+# CHECK-NEXT:],
+# CHECK-NEXT:[
+# CHECK-NEXT:  "entity.name.type.enum.cpp"
+# CHECK-NEXT:],
+# CHECK-NEXT:[
+# CHECK-NEXT:  "variable.other.enummember.cpp"
+# CHECK-NEXT:],
+# CHECK-NEXT:[
+# CHECK-NEXT:  "entity.name.namespace.cpp"
+# CHECK-NEXT:],
+# CHECK-NEXT:[
+# CHECK-NEXT:  "entity.name.type.template.cpp"
+# CHECK-NEXT:]
+# CHECK-NEXT:  ]
+# CHECK-NEXT:}
+# CHECK-NEXT:  },
+---
+{"jsonrpc":"2.0","id":3,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}
Index: clang-tools-extra/clangd/Protocol.h
===
--- clang-tools-extra/clangd/Protocol.h
+++ clang-tools-extra/clangd/Protocol.h
@@ -416,6 +416,10 @@
   /// textDocument.semanticHighlightingCapabilities.semanticHighlighting
   bool SemanticHighlighting = false;
 
+  // Only used because there is no other way to pass the text mate scopes to the
+  // vscode extension other than as experimental server capabilities.
+  bool ExperimentalSemanticHighlighting = false;
+
   /// Supported encodings for LSP character offsets. (clangd extension).
   llvm::Optional> offsetEncoding;
 
Index: clang-tools-extra/clangd/Protocol.cpp
===
--- clang-tools-extra/clangd/Protocol.cpp
+++ clang-tools-extra/clangd/Protocol.cpp
@@ -272,6 +272,12 @@
   const llvm::json::Object *O = Params.getAsObject();
   if (!O)
 return false;
+  if (auto *Experimental = O->getObject("experimental"))
+if (auto *SemanticHighlighting =
+Experimental->getObject("semanticHighlightingCapabilities"))
+  if (auto SemanticHighlightingSupport =
+  SemanticHighlighting->getBoolean("semanticHighlighting"))
+R.ExperimentalSemanticHighlighting = *SemanticHighlightingSupport;
   if (auto *TextDoc

[PATCH] D64922: [clangd] Added option to enable semantic highlighting via an experimental capability

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 210559.
jvikstrom added a comment.

Made Protocol comment be doxygen.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64922/new/

https://reviews.llvm.org/D64922

Files:
  clang-tools-extra/clangd/ClangdLSPServer.cpp
  clang-tools-extra/clangd/Protocol.cpp
  clang-tools-extra/clangd/Protocol.h
  clang-tools-extra/clangd/test/experimental-semantic-highlighting.test
  clang-tools-extra/clangd/test/semantic-highlighting.test

Index: clang-tools-extra/clangd/test/semantic-highlighting.test
===
--- clang-tools-extra/clangd/test/semantic-highlighting.test
+++ clang-tools-extra/clangd/test/semantic-highlighting.test
@@ -2,7 +2,7 @@
 {"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{"textDocument":{"semanticHighlightingCapabilities":{"semanticHighlighting":true}}},"trace":"off"}}
 ---
 #  CHECK:  "id": 0,
-# CHECK:  "semanticHighlighting": {
+#  CHECK:  "semanticHighlighting": {
 # CHECK-NEXT:"scopes": [
 # CHECK-NEXT:  [
 # CHECK-NEXT:"variable.other.cpp"
Index: clang-tools-extra/clangd/test/experimental-semantic-highlighting.test
===
--- /dev/null
+++ clang-tools-extra/clangd/test/experimental-semantic-highlighting.test
@@ -0,0 +1,41 @@
+# RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s
+{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{"experimental":{"semanticHighlightingCapabilities":{"semanticHighlighting":true}}},"trace":"off"}}
+---
+#  CHECK:  "id": 0,
+#  CHECK:  "experimental": {
+# CHECK-NEXT:"semanticHighlighting": {
+# CHECK-NEXT:  "scopes": [
+# CHECK-NEXT:[
+# CHECK-NEXT:  "variable.other.cpp"
+# CHECK-NEXT:],
+# CHECK-NEXT:[
+# CHECK-NEXT:  "entity.name.function.cpp"
+# CHECK-NEXT:],
+# CHECK-NEXT:[
+# CHECK-NEXT:  "entity.name.function.method.cpp"
+# CHECK-NEXT:],
+# CHECK-NEXT:[
+# CHECK-NEXT:  "variable.other.field.cpp"
+# CHECK-NEXT:],
+# CHECK-NEXT:[
+# CHECK-NEXT:  "entity.name.type.class.cpp"
+# CHECK-NEXT:],
+# CHECK-NEXT:[
+# CHECK-NEXT:  "entity.name.type.enum.cpp"
+# CHECK-NEXT:],
+# CHECK-NEXT:[
+# CHECK-NEXT:  "variable.other.enummember.cpp"
+# CHECK-NEXT:],
+# CHECK-NEXT:[
+# CHECK-NEXT:  "entity.name.namespace.cpp"
+# CHECK-NEXT:],
+# CHECK-NEXT:[
+# CHECK-NEXT:  "entity.name.type.template.cpp"
+# CHECK-NEXT:]
+# CHECK-NEXT:  ]
+# CHECK-NEXT:}
+# CHECK-NEXT:  },
+---
+{"jsonrpc":"2.0","id":3,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}
Index: clang-tools-extra/clangd/Protocol.h
===
--- clang-tools-extra/clangd/Protocol.h
+++ clang-tools-extra/clangd/Protocol.h
@@ -416,6 +416,10 @@
   /// textDocument.semanticHighlightingCapabilities.semanticHighlighting
   bool SemanticHighlighting = false;
 
+  /// Only used because there is no other way to pass the text mate scopes to
+  /// the vscode extension other than as experimental server capabilities.
+  bool ExperimentalSemanticHighlighting = false;
+
   /// Supported encodings for LSP character offsets. (clangd extension).
   llvm::Optional> offsetEncoding;
 
Index: clang-tools-extra/clangd/Protocol.cpp
===
--- clang-tools-extra/clangd/Protocol.cpp
+++ clang-tools-extra/clangd/Protocol.cpp
@@ -272,6 +272,12 @@
   const llvm::json::Object *O = Params.getAsObject();
   if (!O)
 return false;
+  if (auto *Experimental = O->getObject("experimental"))
+if (auto *SemanticHighlighting =
+Experimental->getObject("semanticHighlightingCapabilities"))
+  if (auto SemanticHighlightingSupport =
+  SemanticHighlighting->getBoolean("semanticHighlighting"))
+R.ExperimentalSemanticHighlighting = *SemanticHighlightingSupport;
   if (auto *TextDocument = O->getObject("textDocument")) {
 if (auto *SemanticHighlighting =
 TextDocument->getObject("semanticHighlightingCapabilities")) {
Index: clang-tools-extra/clangd/ClangdLSPServer.cpp
===
--- clang-tools-extra/clangd/ClangdLSPServer.cpp
+++ clang-tools-extra/clangd/ClangdLSPServer.cpp
@@ -341,7 +341,8 @@
*NegotiatedOffsetEncoding);
 
   ClangdServerOpts.SemanticHighlighting =
-  Params.capabilities.SemanticHighlighting;
+  Params.capabilities.SemanticHighlighti

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom marked 6 inline comments as done.
jvikstrom added inline comments.



Comment at: clang-tools-extra/clangd/ClangdLSPServer.cpp:458
+// edit there are stale previous highlightings.
+std::lock_guard Lock(HighlightingsMutex);
+FileToHighlightings.erase(File);

ilya-biryukov wrote:
> Should can't we handle this on `didClose` instead?
We are removing in didClose but the problem is that there is a race condition 
(I think).

If someone does some edits and closes the document right after, the 
highlightings for the final edit might finish being generated after the 
FileToHighlightings have earsed the highlightings for the file. So next time 
when opening the file we will have those final highlightings that were 
generated for the last edit in the map. 
I don't really know how we could handle this in didClose without having another 
map and with an open/closed bit and checking that every time we generate new 
highlightings. But we'd still have to set the bit to be open every didOpen so I 
don't really see the benefit.

However I've head that ASTWorked is synchronous for a single file, is that the 
case for the didClose call as well? Because in that case there is no race 
condition.



Comment at: clang-tools-extra/clangd/SemanticHighlighting.cpp:276
+  ArrayRef OldHighlightings) {
+  // FIXME: There's an edge case when tokens span multiple lines. If the first
+  // token on the line started on a line above the current one and the rest of

ilya-biryukov wrote:
> ilya-biryukov wrote:
> > Do you have any ideas on how we can fix this?
> > 
> > I would simply split those tokens into multiple tokens of the same kind at 
> > newlines boundaries, but maybe there are other ways to handle this.
> > 
> > In any case, could you put a suggested way to fix this (in case someone 
> > will want to pick it up, they'll have a starting point)
> NIT: add assertions that highlightings are sorted.
Yeah, I would split the tokens into multiple lines as well. Or enforce that a 
token is single line and handle it in addToken in the collector. (i.e. change 
the HighlightingToken struct)

It's a bit annoying to solve because we'd have to get the line lengths of every 
line that the multiline length token covers when doing that.



Comment at: clang-tools-extra/clangd/SemanticHighlighting.h:79
+std::vector
+diffHighlightings(ArrayRef NewHighlightings,
+  ArrayRef OldHighlightings);

ilya-biryukov wrote:
> Are we locked into the line-based diff implementation?
> It works nicely while editing inside the same line, but seem to do a bad job 
> on a common case of inserting/removing lines.
> 
> Does the protocol have a way to communicate this cleanly?
We are looked into some kind of line based diff implementation as the protocol 
sends token line by line. 
There should be away of solving the inserting/removing lines, but I'll have a 
look at that in a follow up. 
Theia seems to do a good job of moving tokens to where they should be 
automatically when inserting a new line though. I want to be able to see how 
vscode handles it first as well though before.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 210567.
jvikstrom edited the summary of this revision.
jvikstrom added a comment.

Added additional note to FIXME on how to solve and also move(ing) Old.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475

Files:
  clang-tools-extra/clangd/ClangdLSPServer.cpp
  clang-tools-extra/clangd/ClangdLSPServer.h
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/semantic-highlighting.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,9 +29,12 @@
   return Tokens;
 }
 
-void checkHighlightings(llvm::StringRef Code) {
-  Annotations Test(Code);
+std::vector getActualTokens(Annotations &Test) {
   auto AST = TestTU::withCode(Test.code()).build();
+  return getSemanticHighlightings(AST);
+}
+
+std::vector getExpectedTokens(Annotations &Test) {
   static const std::map KindToString{
   {HighlightingKind::Variable, "Variable"},
   {HighlightingKind::Function, "Function"},
@@ -48,9 +51,43 @@
 Test.ranges(KindString.second), KindString.first);
 ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
   }
+  llvm::sort(ExpectedTokens);
+  return ExpectedTokens;
+}
 
-  auto ActualTokens = getSemanticHighlightings(AST);
-  EXPECT_THAT(ActualTokens, testing::UnorderedElementsAreArray(ExpectedTokens));
+std::vector getExpectedEmptyLines(Annotations &Test) {
+  auto EmptyRanges = Test.ranges("Empty");
+  std::vector EmptyLines;
+  EmptyLines.reserve(EmptyRanges.size());
+  for (const auto &EmptyRange : EmptyRanges)
+EmptyLines.push_back(EmptyRange.start.line);
+  return EmptyLines;
+}
+
+void checkHighlightings(llvm::StringRef Code) {
+  Annotations Test(Code);
+  EXPECT_THAT(getActualTokens(Test),
+  getExpectedTokens(Test));
+}
+
+void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) {
+  Annotations OldTest(OldCode);
+  Annotations NewTest(NewCode);
+  std::vector ExpectedLinePairHighlighting;
+  for (int Line : getExpectedEmptyLines(NewTest))
+ExpectedLinePairHighlighting.push_back({Line, {}});
+  std::map> ExpectedLines;
+  for (const HighlightingToken &Token : getExpectedTokens(NewTest))
+ExpectedLines[Token.R.start.line].push_back(Token);
+  for (auto &LineTokens : ExpectedLines) {
+ExpectedLinePairHighlighting.push_back(
+{LineTokens.first, LineTokens.second});
+  }
+
+  std::vector ActualDiffed =
+  diffHighlightings(getActualTokens(NewTest), getActualTokens(OldTest));
+  EXPECT_THAT(ActualDiffed,
+  testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting));
 }
 
 TEST(SemanticHighlighting, GetsCorrectTokens) {
@@ -251,21 +288,110 @@
 return Pos;
   };
 
-  std::vector Tokens{
-  {HighlightingKind::Variable,
-Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
-  {HighlightingKind::Function,
-Range{CreatePosition(3, 4), CreatePosition(3, 7)}},
-  {HighlightingKind::Variable,
-Range{CreatePosition(1, 1), CreatePosition(1, 5)}}};
+  std::vector Tokens{
+  {3,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
+{HighlightingKind::Function,
+ Range{CreatePosition(3, 4), CreatePosition(3, 7),
+  {1,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(1, 1), CreatePosition(1, 5)};
   std::vector ActualResults =
   toSemanticHighlightingInformation(Tokens);
   std::vector ExpectedResults = {
-  {1, "AQAEAAA="},
-  {3, "CAAEAAAEAAMAAQ=="}};
+  {3, "CAAEAAAEAAMAAQ=="}, {1, "AQAEAAA="}};
   EXPECT_EQ(ActualResults, ExpectedResults);
 }
 
+TEST(SemanticHighlighting, HighlightingDiffer) {
+  struct {
+llvm::StringRef OldCode;
+llvm::StringRef NewCode;
+  } TestCases[]{{
+  R"cpp(
+int A
+double B;
+struct C {};
+  )cpp",
+  R"cpp(
+int A;
+double B;
+struct C {};
+  )cpp"},
+{
+  R"cpp(
+struct Alpha {
+  double SomeVariable = 9483.301;
+};
+struct Beta {};
+int A = 121;
+Alpha Var;
+  )cpp",
+  R"cpp(
+struct Alpha {
+  double SomeVariable = 9483.301;
+};
+struct Beta   {}; // Some comment
+$Variable[[intA]] = 121;
+$Class[[Beta]] $Variable[[Var]];
+  )cpp"},
+{
+  R"cpp(
+int A = 121; int B;
+  )cpp",
+  R"cpp(
+$Variable[[intA]] = 121; int $Variable[[B]];
+  )c

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 210601.
jvikstrom added a comment.

Remove braces.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475

Files:
  clang-tools-extra/clangd/ClangdLSPServer.cpp
  clang-tools-extra/clangd/ClangdLSPServer.h
  clang-tools-extra/clangd/ClangdServer.cpp
  clang-tools-extra/clangd/ClangdServer.h
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/semantic-highlighting.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,9 +29,7 @@
   return Tokens;
 }
 
-void checkHighlightings(llvm::StringRef Code) {
-  Annotations Test(Code);
-  auto AST = TestTU::withCode(Test.code()).build();
+std::vector getExpectedTokens(Annotations &Test) {
   static const std::map KindToString{
   {HighlightingKind::Variable, "Variable"},
   {HighlightingKind::Function, "Function"},
@@ -48,9 +46,43 @@
 Test.ranges(KindString.second), KindString.first);
 ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
   }
+  llvm::sort(ExpectedTokens);
+  return ExpectedTokens;
+}
+
+void checkHighlightings(llvm::StringRef Code) {
+  Annotations Test(Code);
+  auto AST = TestTU::withCode(Test.code()).build();
+  std::vector ActualTokens = getSemanticHighlightings(AST);
+  EXPECT_THAT(ActualTokens, getExpectedTokens(Test));
+}
+
+void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode,
+   llvm::ArrayRef DiffedLines) {
+  Annotations OldTest(OldCode);
+  Annotations NewTest(NewCode);
+  std::vector OldTokens = getExpectedTokens(OldTest);
+  std::vector NewTokens = getExpectedTokens(NewTest);
+
+  std::map> ExpectedLines;
+  for (int Line : DiffedLines) {
+ExpectedLines[Line]; // Default initialize to an empty line. Tokens are
+ // inserted on these lines later.
+  }
+  std::vector ExpectedLinePairHighlighting;
+  for (const HighlightingToken &Token : NewTokens) {
+auto It = ExpectedLines.find(Token.R.start.line);
+if (It != ExpectedLines.end())
+  It->second.push_back(Token);
+  }
+  for (auto &LineTokens : ExpectedLines)
+ExpectedLinePairHighlighting.push_back(
+{LineTokens.first, LineTokens.second});
 
-  auto ActualTokens = getSemanticHighlightings(AST);
-  EXPECT_THAT(ActualTokens, testing::UnorderedElementsAreArray(ExpectedTokens));
+  std::vector ActualDiffed =
+  diffHighlightings(NewTokens, OldTokens, NewCode.count('\n'));
+  EXPECT_THAT(ActualDiffed,
+  testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting));
 }
 
 TEST(SemanticHighlighting, GetsCorrectTokens) {
@@ -225,8 +257,9 @@
 std::atomic Count = {0};
 
 void onDiagnosticsReady(PathRef, std::vector) override {}
-void onHighlightingsReady(
-PathRef File, std::vector Highlightings) override {
+void onHighlightingsReady(PathRef File,
+  std::vector Highlightings,
+  int NLines) override {
   ++Count;
 }
   };
@@ -251,21 +284,135 @@
 return Pos;
   };
 
-  std::vector Tokens{
-  {HighlightingKind::Variable,
-Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
-  {HighlightingKind::Function,
-Range{CreatePosition(3, 4), CreatePosition(3, 7)}},
-  {HighlightingKind::Variable,
-Range{CreatePosition(1, 1), CreatePosition(1, 5)}}};
+  std::vector Tokens{
+  {3,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
+{HighlightingKind::Function,
+ Range{CreatePosition(3, 4), CreatePosition(3, 7),
+  {1,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(1, 1), CreatePosition(1, 5)};
   std::vector ActualResults =
   toSemanticHighlightingInformation(Tokens);
   std::vector ExpectedResults = {
-  {1, "AQAEAAA="},
-  {3, "CAAEAAAEAAMAAQ=="}};
+  {3, "CAAEAAAEAAMAAQ=="}, {1, "AQAEAAA="}};
   EXPECT_EQ(ActualResults, ExpectedResults);
 }
 
+TEST(SemanticHighlighting, HighlightingDiffer) {
+  struct {
+llvm::StringRef OldCode;
+llvm::StringRef NewCode;
+std::vector DiffedLines;
+  } TestCases[]{
+{
+  R"cpp(
+$Variable[[A]]
+$Class[[B]]
+$Function[[C]]
+  )cpp",
+  R"cpp(
+$Variable[[A]]
+$Class[[D]]
+$Function[[C]]
+  )cpp",
+  {}},
+{
+  R"cpp(
+$Class[[C]]
+$Field[[F]]
+$Variable[[V]]
+$Class[[C]] $Variable[[V

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 210600.
jvikstrom marked 7 inline comments as done.
jvikstrom added a comment.

Address comments and fixed bug where we'd send diffs outside of the file.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475

Files:
  clang-tools-extra/clangd/ClangdLSPServer.cpp
  clang-tools-extra/clangd/ClangdLSPServer.h
  clang-tools-extra/clangd/ClangdServer.cpp
  clang-tools-extra/clangd/ClangdServer.h
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/semantic-highlighting.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,9 +29,7 @@
   return Tokens;
 }
 
-void checkHighlightings(llvm::StringRef Code) {
-  Annotations Test(Code);
-  auto AST = TestTU::withCode(Test.code()).build();
+std::vector getExpectedTokens(Annotations &Test) {
   static const std::map KindToString{
   {HighlightingKind::Variable, "Variable"},
   {HighlightingKind::Function, "Function"},
@@ -48,9 +46,43 @@
 Test.ranges(KindString.second), KindString.first);
 ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
   }
+  llvm::sort(ExpectedTokens);
+  return ExpectedTokens;
+}
+
+void checkHighlightings(llvm::StringRef Code) {
+  Annotations Test(Code);
+  auto AST = TestTU::withCode(Test.code()).build();
+  std::vector ActualTokens = getSemanticHighlightings(AST);
+  EXPECT_THAT(ActualTokens, getExpectedTokens(Test));
+}
+
+void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode,
+   llvm::ArrayRef DiffedLines) {
+  Annotations OldTest(OldCode);
+  Annotations NewTest(NewCode);
+  std::vector OldTokens = getExpectedTokens(OldTest);
+  std::vector NewTokens = getExpectedTokens(NewTest);
+
+  std::map> ExpectedLines;
+  for (int Line : DiffedLines) {
+ExpectedLines[Line]; // Default initialize to an empty line. Tokens are
+ // inserted on these lines later.
+  }
+  std::vector ExpectedLinePairHighlighting;
+  for (const HighlightingToken &Token : NewTokens) {
+auto It = ExpectedLines.find(Token.R.start.line);
+if (It != ExpectedLines.end())
+  It->second.push_back(Token);
+  }
+  for (auto &LineTokens : ExpectedLines)
+ExpectedLinePairHighlighting.push_back(
+{LineTokens.first, LineTokens.second});
 
-  auto ActualTokens = getSemanticHighlightings(AST);
-  EXPECT_THAT(ActualTokens, testing::UnorderedElementsAreArray(ExpectedTokens));
+  std::vector ActualDiffed =
+  diffHighlightings(NewTokens, OldTokens, NewCode.count('\n'));
+  EXPECT_THAT(ActualDiffed,
+  testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting));
 }
 
 TEST(SemanticHighlighting, GetsCorrectTokens) {
@@ -225,8 +257,9 @@
 std::atomic Count = {0};
 
 void onDiagnosticsReady(PathRef, std::vector) override {}
-void onHighlightingsReady(
-PathRef File, std::vector Highlightings) override {
+void onHighlightingsReady(PathRef File,
+  std::vector Highlightings,
+  int NLines) override {
   ++Count;
 }
   };
@@ -251,21 +284,137 @@
 return Pos;
   };
 
-  std::vector Tokens{
-  {HighlightingKind::Variable,
-Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
-  {HighlightingKind::Function,
-Range{CreatePosition(3, 4), CreatePosition(3, 7)}},
-  {HighlightingKind::Variable,
-Range{CreatePosition(1, 1), CreatePosition(1, 5)}}};
+  std::vector Tokens{
+  {3,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
+{HighlightingKind::Function,
+ Range{CreatePosition(3, 4), CreatePosition(3, 7),
+  {1,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(1, 1), CreatePosition(1, 5)};
   std::vector ActualResults =
   toSemanticHighlightingInformation(Tokens);
   std::vector ExpectedResults = {
-  {1, "AQAEAAA="},
-  {3, "CAAEAAAEAAMAAQ=="}};
+  {3, "CAAEAAAEAAMAAQ=="}, {1, "AQAEAAA="}};
   EXPECT_EQ(ActualResults, ExpectedResults);
 }
 
+TEST(SemanticHighlighting, HighlightingDiffer) {
+  struct {
+llvm::StringRef OldCode;
+llvm::StringRef NewCode;
+std::vector DiffedLines;
+  } TestCases[]{
+{
+  R"cpp(
+$Variable[[A]]
+$Class[[B]]
+$Function[[C]]
+  )cpp",
+  R"cpp(
+$Variable[[A]]
+$Class[[D]]
+$Function[[C]]
+  )cpp",
+  {}},
+{
+  R"c

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 210606.
jvikstrom added a comment.

Fixed test case that was actually (kinda) broken even though it didn't fail.

(The last two edits in semantic-highlighting test. They removed the `= 2` part 
and did not insert an extra `;` which produced invalid code. It was still fine 
because the highlightings that were supposed to be generated were still 
generated. This just makes the code not be invalid)


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475

Files:
  clang-tools-extra/clangd/ClangdLSPServer.cpp
  clang-tools-extra/clangd/ClangdLSPServer.h
  clang-tools-extra/clangd/ClangdServer.cpp
  clang-tools-extra/clangd/ClangdServer.h
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/semantic-highlighting.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,9 +29,7 @@
   return Tokens;
 }
 
-void checkHighlightings(llvm::StringRef Code) {
-  Annotations Test(Code);
-  auto AST = TestTU::withCode(Test.code()).build();
+std::vector getExpectedTokens(Annotations &Test) {
   static const std::map KindToString{
   {HighlightingKind::Variable, "Variable"},
   {HighlightingKind::Function, "Function"},
@@ -48,9 +46,43 @@
 Test.ranges(KindString.second), KindString.first);
 ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
   }
+  llvm::sort(ExpectedTokens);
+  return ExpectedTokens;
+}
+
+void checkHighlightings(llvm::StringRef Code) {
+  Annotations Test(Code);
+  auto AST = TestTU::withCode(Test.code()).build();
+  std::vector ActualTokens = getSemanticHighlightings(AST);
+  EXPECT_THAT(ActualTokens, getExpectedTokens(Test));
+}
+
+void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode,
+   llvm::ArrayRef DiffedLines) {
+  Annotations OldTest(OldCode);
+  Annotations NewTest(NewCode);
+  std::vector OldTokens = getExpectedTokens(OldTest);
+  std::vector NewTokens = getExpectedTokens(NewTest);
+
+  std::map> ExpectedLines;
+  for (int Line : DiffedLines) {
+ExpectedLines[Line]; // Default initialize to an empty line. Tokens are
+ // inserted on these lines later.
+  }
+  std::vector ExpectedLinePairHighlighting;
+  for (const HighlightingToken &Token : NewTokens) {
+auto It = ExpectedLines.find(Token.R.start.line);
+if (It != ExpectedLines.end())
+  It->second.push_back(Token);
+  }
+  for (auto &LineTokens : ExpectedLines)
+ExpectedLinePairHighlighting.push_back(
+{LineTokens.first, LineTokens.second});
 
-  auto ActualTokens = getSemanticHighlightings(AST);
-  EXPECT_THAT(ActualTokens, testing::UnorderedElementsAreArray(ExpectedTokens));
+  std::vector ActualDiffed =
+  diffHighlightings(NewTokens, OldTokens, NewCode.count('\n'));
+  EXPECT_THAT(ActualDiffed,
+  testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting));
 }
 
 TEST(SemanticHighlighting, GetsCorrectTokens) {
@@ -225,8 +257,9 @@
 std::atomic Count = {0};
 
 void onDiagnosticsReady(PathRef, std::vector) override {}
-void onHighlightingsReady(
-PathRef File, std::vector Highlightings) override {
+void onHighlightingsReady(PathRef File,
+  std::vector Highlightings,
+  int NLines) override {
   ++Count;
 }
   };
@@ -251,21 +284,135 @@
 return Pos;
   };
 
-  std::vector Tokens{
-  {HighlightingKind::Variable,
-Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
-  {HighlightingKind::Function,
-Range{CreatePosition(3, 4), CreatePosition(3, 7)}},
-  {HighlightingKind::Variable,
-Range{CreatePosition(1, 1), CreatePosition(1, 5)}}};
+  std::vector Tokens{
+  {3,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
+{HighlightingKind::Function,
+ Range{CreatePosition(3, 4), CreatePosition(3, 7),
+  {1,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(1, 1), CreatePosition(1, 5)};
   std::vector ActualResults =
   toSemanticHighlightingInformation(Tokens);
   std::vector ExpectedResults = {
-  {1, "AQAEAAA="},
-  {3, "CAAEAAAEAAMAAQ=="}};
+  {3, "CAAEAAAEAAMAAQ=="}, {1, "AQAEAAA="}};
   EXPECT_EQ(ActualResults, ExpectedResults);
 }
 
+TEST(SemanticHighlighting, HighlightingDiffer) {
+  struct {
+llvm::StringRef OldCode;
+llvm::StringRef NewCode;
+std::vector DiffedLines;
+  

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 210613.
jvikstrom added a comment.

Removed one lock from onSemanticHighlighting.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475

Files:
  clang-tools-extra/clangd/ClangdLSPServer.cpp
  clang-tools-extra/clangd/ClangdLSPServer.h
  clang-tools-extra/clangd/ClangdServer.cpp
  clang-tools-extra/clangd/ClangdServer.h
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/semantic-highlighting.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,9 +29,7 @@
   return Tokens;
 }
 
-void checkHighlightings(llvm::StringRef Code) {
-  Annotations Test(Code);
-  auto AST = TestTU::withCode(Test.code()).build();
+std::vector getExpectedTokens(Annotations &Test) {
   static const std::map KindToString{
   {HighlightingKind::Variable, "Variable"},
   {HighlightingKind::Function, "Function"},
@@ -48,9 +46,43 @@
 Test.ranges(KindString.second), KindString.first);
 ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
   }
+  llvm::sort(ExpectedTokens);
+  return ExpectedTokens;
+}
+
+void checkHighlightings(llvm::StringRef Code) {
+  Annotations Test(Code);
+  auto AST = TestTU::withCode(Test.code()).build();
+  std::vector ActualTokens = getSemanticHighlightings(AST);
+  EXPECT_THAT(ActualTokens, getExpectedTokens(Test));
+}
+
+void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode,
+   llvm::ArrayRef DiffedLines) {
+  Annotations OldTest(OldCode);
+  Annotations NewTest(NewCode);
+  std::vector OldTokens = getExpectedTokens(OldTest);
+  std::vector NewTokens = getExpectedTokens(NewTest);
+
+  std::map> ExpectedLines;
+  for (int Line : DiffedLines) {
+ExpectedLines[Line]; // Default initialize to an empty line. Tokens are
+ // inserted on these lines later.
+  }
+  std::vector ExpectedLinePairHighlighting;
+  for (const HighlightingToken &Token : NewTokens) {
+auto It = ExpectedLines.find(Token.R.start.line);
+if (It != ExpectedLines.end())
+  It->second.push_back(Token);
+  }
+  for (auto &LineTokens : ExpectedLines)
+ExpectedLinePairHighlighting.push_back(
+{LineTokens.first, LineTokens.second});
 
-  auto ActualTokens = getSemanticHighlightings(AST);
-  EXPECT_THAT(ActualTokens, testing::UnorderedElementsAreArray(ExpectedTokens));
+  std::vector ActualDiffed =
+  diffHighlightings(NewTokens, OldTokens, NewCode.count('\n'));
+  EXPECT_THAT(ActualDiffed,
+  testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting));
 }
 
 TEST(SemanticHighlighting, GetsCorrectTokens) {
@@ -225,8 +257,9 @@
 std::atomic Count = {0};
 
 void onDiagnosticsReady(PathRef, std::vector) override {}
-void onHighlightingsReady(
-PathRef File, std::vector Highlightings) override {
+void onHighlightingsReady(PathRef File,
+  std::vector Highlightings,
+  int NLines) override {
   ++Count;
 }
   };
@@ -251,21 +284,135 @@
 return Pos;
   };
 
-  std::vector Tokens{
-  {HighlightingKind::Variable,
-Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
-  {HighlightingKind::Function,
-Range{CreatePosition(3, 4), CreatePosition(3, 7)}},
-  {HighlightingKind::Variable,
-Range{CreatePosition(1, 1), CreatePosition(1, 5)}}};
+  std::vector Tokens{
+  {3,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
+{HighlightingKind::Function,
+ Range{CreatePosition(3, 4), CreatePosition(3, 7),
+  {1,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(1, 1), CreatePosition(1, 5)};
   std::vector ActualResults =
   toSemanticHighlightingInformation(Tokens);
   std::vector ExpectedResults = {
-  {1, "AQAEAAA="},
-  {3, "CAAEAAAEAAMAAQ=="}};
+  {3, "CAAEAAAEAAMAAQ=="}, {1, "AQAEAAA="}};
   EXPECT_EQ(ActualResults, ExpectedResults);
 }
 
+TEST(SemanticHighlighting, HighlightingDiffer) {
+  struct {
+llvm::StringRef OldCode;
+llvm::StringRef NewCode;
+std::vector DiffedLines;
+  } TestCases[]{
+{
+  R"cpp(
+$Variable[[A]]
+$Class[[B]]
+$Function[[C]]
+  )cpp",
+  R"cpp(
+$Variable[[A]]
+$Class[[D]]
+$Function[[C]]
+  )cpp",
+  {}},
+{
+  R"cpp(
+$Class[[C]]
+$Field[[F]]
+$Variable[[V]]
+ 

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom added inline comments.



Comment at: clang-tools-extra/clangd/SemanticHighlighting.cpp:292
+  /*length*/0UL);
+  auto NewEnd = NewHighlightings.end();
+  auto OldEnd = OldHighlightings.end();

ilya-biryukov wrote:
> NIT: maybe just inline `NewEnd` and `OldEnd` to save a few lines?
Wouldn't that violate code style: 
https://llvm.org/docs/CodingStandards.html#don-t-evaluate-end-every-time-through-a-loop
 ?
Or maybe that doesn't matter? (I have no preference either way, just picked 
this way as I could remeber reading from the styleguide)



Comment at: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp:307
 
+TEST(SemanticHighlighting, HighlightingDiffer) {
+  struct {

ilya-biryukov wrote:
> Can we test this in a more direct manner by specifying:
> 1. annotated input for old highlightings,
> 2. annotated input for new highlightings,
> 3. the expected diff?
> 
> The resulting tests don't have to be real C++ then, allowing to express what 
> we're testing in a more direct manner.
> ```
> {/*Old*/ "$Variable[[a]]", /*New*/ "$Class[[a]]", /*Diff*/ "{{/*line */0, 
> "$Class[[a]]"}}
> ```
> 
> It also seems that the contents of the lines could even be checked 
> automatically (they should be equal to the corresponding line from `/*New*/`, 
> right?), that leaves us with even simpler inputs:
> ```
> {/*Old*/ "$Variable[[a]]", /*New*/ "$Class[[a]]", /*DiffLines*/ "{0}}
> ```
That's a great idea on how to write these tests.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 211890.
jvikstrom marked 2 inline comments as done.
jvikstrom added a comment.

Made tests more readable. Updated to work with rL366577 
.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475

Files:
  clang-tools-extra/clangd/ClangdLSPServer.cpp
  clang-tools-extra/clangd/ClangdLSPServer.h
  clang-tools-extra/clangd/ClangdServer.cpp
  clang-tools-extra/clangd/ClangdServer.h
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/semantic-highlighting.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,9 +29,7 @@
   return Tokens;
 }
 
-void checkHighlightings(llvm::StringRef Code) {
-  Annotations Test(Code);
-  auto AST = TestTU::withCode(Test.code()).build();
+std::vector getExpectedTokens(Annotations &Test) {
   static const std::map KindToString{
   {HighlightingKind::Variable, "Variable"},
   {HighlightingKind::Function, "Function"},
@@ -48,10 +46,42 @@
 Test.ranges(KindString.second), KindString.first);
 ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
   }
+  llvm::sort(ExpectedTokens);
+  return ExpectedTokens;
+}
 
-  auto ActualTokens = getSemanticHighlightings(AST);
-  EXPECT_THAT(ActualTokens, testing::UnorderedElementsAreArray(ExpectedTokens))
-   << "Inputs is:\n" << Code;
+void checkHighlightings(llvm::StringRef Code) {
+  Annotations Test(Code);
+  auto AST = TestTU::withCode(Test.code()).build();
+  std::vector ActualTokens = getSemanticHighlightings(AST);
+  EXPECT_THAT(ActualTokens, getExpectedTokens(Test));
+}
+
+void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) {
+  Annotations OldTest(OldCode);
+  Annotations NewTest(NewCode);
+  std::vector OldTokens = getExpectedTokens(OldTest);
+  std::vector NewTokens = getExpectedTokens(NewTest);
+
+  std::map> ExpectedLines;
+  for (const Position &Point : NewTest.points()) {
+ExpectedLines[Point.line]; // Default initialize to an empty line. Tokens
+   // are inserted on these lines later.
+  }
+  std::vector ExpectedLinePairHighlighting;
+  for (const HighlightingToken &Token : NewTokens) {
+auto It = ExpectedLines.find(Token.R.start.line);
+if (It != ExpectedLines.end())
+  It->second.push_back(Token);
+  }
+  for (auto &LineTokens : ExpectedLines)
+ExpectedLinePairHighlighting.push_back(
+{LineTokens.first, LineTokens.second});
+
+  std::vector ActualDiffed =
+  diffHighlightings(NewTokens, OldTokens, NewCode.count('\n'));
+  EXPECT_THAT(ActualDiffed,
+  testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting));
 }
 
 TEST(SemanticHighlighting, GetsCorrectTokens) {
@@ -226,8 +256,9 @@
 std::atomic Count = {0};
 
 void onDiagnosticsReady(PathRef, std::vector) override {}
-void onHighlightingsReady(
-PathRef File, std::vector Highlightings) override {
+void onHighlightingsReady(PathRef File,
+  std::vector Highlightings,
+  int NLines) override {
   ++Count;
 }
   };
@@ -252,21 +283,126 @@
 return Pos;
   };
 
-  std::vector Tokens{
-  {HighlightingKind::Variable,
-Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
-  {HighlightingKind::Function,
-Range{CreatePosition(3, 4), CreatePosition(3, 7)}},
-  {HighlightingKind::Variable,
-Range{CreatePosition(1, 1), CreatePosition(1, 5)}}};
+  std::vector Tokens{
+  {3,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
+{HighlightingKind::Function,
+ Range{CreatePosition(3, 4), CreatePosition(3, 7),
+  {1,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(1, 1), CreatePosition(1, 5)};
   std::vector ActualResults =
   toSemanticHighlightingInformation(Tokens);
   std::vector ExpectedResults = {
-  {1, "AQAEAAA="},
-  {3, "CAAEAAAEAAMAAQ=="}};
+  {3, "CAAEAAAEAAMAAQ=="}, {1, "AQAEAAA="}};
   EXPECT_EQ(ActualResults, ExpectedResults);
 }
 
+TEST(SemanticHighlighting, HighlightingDiffer) {
+  struct {
+llvm::StringRef OldCode;
+llvm::StringRef NewCode;
+  } TestCases[]{
+{
+  R"(
+$Variable[[A]]
+$Class[[B]]
+$Function[[C]]
+  )",
+  R"(
+$Variable[[A]]
+$Class[[D]]
+$Function[[C]]
+  )"},
+{
+  R"(
+$Class[[C]]
+ 

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom added a comment.

In D64475#1593481 , @ilya-biryukov 
wrote:

> The fix for a race condition on remove has landed in rL366577 
> , this revision would need a small update 
> after it.


Fixed to work with that patch.
Should the diffing be moved somewhere else that is not inside the publish 
function though? That would require moving the state outside the LSP server 
though and handle the onClose/onOpen events somewhere else though. 
Maybe it's ok to keep the diffing in the publish lock?




Comment at: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp:307
+llvm::StringRef NewCode;
+std::vector DiffedLines;
+  } TestCases[]{

ilya-biryukov wrote:
> @hokein rightfully pointed out that mentioning all changed lines makes the 
> tests unreadable.
> An alternative idea we all came up with is to force people to put `^` on each 
> of the changed lines inside the `NewCode`, i.e.
> 
> ```
> {/*Before*/ R"(
>   $Var[[a]]
>   $Func[[b]]
> "),
>  /*After*/ R"(
>   $Var[[a]]
>  ^$Func[[b]]
> )"} // and no list of lines is needed!
> ```
> 
> Could we do that here?
> 
> One interesting case that we can't test this way to removing lines from the 
> end of the file. But for that particular case, could we just write a separate 
> test case?
We don't want to send diffs that are beyond the end of the file. There is a 
testcase for that as well (the count of newlines in the new code is sent to the 
differ as the number of lines in the file).



Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D65395: [clangd] Added a skeleton for a semantic highlighting feature to the vscode extension.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom created this revision.
jvikstrom added reviewers: hokein, ilya-biryukov.
Herald added subscribers: cfe-commits, kadircet, arphaman, jkorous, MaskRay.
Herald added a project: clang.

This adds a SemanticHighlighting feature to the vscode extension that does not 
do anything. It adds the notification handler to the clangd client and 
registers the feature as an experimental feature sending the relevant data to 
the clangd server.
The notification handler does not currently do anything if/when clangd sends a 
semantic highlighting notification.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D65395

Files:
  clang-tools-extra/clangd/clients/clangd-vscode/.gitignore
  clang-tools-extra/clangd/clients/clangd-vscode/src/SemanticHighlighting.ts
  clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts

Index: clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
===
--- clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
+++ clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
@@ -1,5 +1,6 @@
 import * as vscode from 'vscode';
 import * as vscodelc from 'vscode-languageclient';
+import { SemanticHighlighting } from './SemanticHighlighting';
 
 /**
  * Method to get workspace configuration option
@@ -12,9 +13,9 @@
 }
 
 namespace SwitchSourceHeaderRequest {
-export const type =
-new vscodelc.RequestType('textDocument/switchSourceHeader');
+export const type =
+new vscodelc.RequestType('textDocument/switchSourceHeader');
 }
 
 class FileStatus {
@@ -32,8 +33,8 @@
 const path = vscode.window.activeTextEditor.document.fileName;
 const status = this.statuses.get(path);
 if (!status) {
-  this.statusBarItem.hide();
-  return;
+this.statusBarItem.hide();
+return;
 }
 this.statusBarItem.text = `clangd: ` + status.state;
 this.statusBarItem.show();
@@ -73,25 +74,30 @@
 // However, VSCode does not have CUDA as a supported language yet, so we
 // cannot add a corresponding activationEvent for CUDA files and clangd will
 // *not* load itself automatically on '.cu' files.
-const cudaFilePattern: string = '**/*.{' +['cu'].join()+ '}';
+const cudaFilePattern: string = '**/*.{' + ['cu'].join() + '}';
 const clientOptions: vscodelc.LanguageClientOptions = {
 // Register the server for c-family and cuda files.
 documentSelector: [
 { scheme: 'file', language: 'c' },
 { scheme: 'file', language: 'cpp' },
-{ scheme: 'file', language: 'objective-c'},
-{ scheme: 'file', language: 'objective-cpp'},
+{ scheme: 'file', language: 'objective-c' },
+{ scheme: 'file', language: 'objective-cpp' },
 { scheme: 'file', pattern: cudaFilePattern },
 ],
 synchronize: !syncFileEvents ? undefined : {
-// FIXME: send sync file events when clangd provides implemenatations.
+// FIXME: send sync file events when clangd provides implemenatations.
 },
 initializationOptions: { clangdFileStatus: true },
 // Do not switch to output window when clangd returns output
 revealOutputChannelOn: vscodelc.RevealOutputChannelOn.Never
 };
 
-const clangdClient = new vscodelc.LanguageClient('Clang Language Server',serverOptions, clientOptions);
+const clangdClient = new vscodelc.LanguageClient('Clang Language Server', serverOptions, clientOptions);
+const semanticHighlightingFeature = new SemanticHighlighting.Feature();
+clangdClient.registerFeature(semanticHighlightingFeature);
+// The notification handler must be registered after the client is ready or the client will crash.
+clangdClient.onReady().then(() => clangdClient.onNotification(SemanticHighlighting.NotificationType, semanticHighlightingFeature.handleNotification));
+
 console.log('Clang Language Server is now active!');
 context.subscriptions.push(clangdClient.start());
 context.subscriptions.push(vscode.commands.registerCommand(
@@ -131,5 +137,5 @@
 // An empty place holder for the activate command, otherwise we'll get an
 // "command is not registered" error.
 context.subscriptions.push(vscode.commands.registerCommand(
-'clangd-vscode.activate', async () => {}));
+'clangd-vscode.activate', async () => { }));
 }
Index: clang-tools-extra/clangd/clients/clangd-vscode/src/SemanticHighlighting.ts
===
--- /dev/null
+++ clang-tools-extra/clangd/clients/clangd-vscode/src/SemanticHighlighting.ts
@@ -0,0 +1,36 @@
+import * as vscodelc from 'vscode-languageclient';
+import { DocumentSelector } from 'vscode-languageclient';
+
+export namespace SemanticHighlighting {
+interface HighlightingLineInformation {
+line: number,
+tokens: string,
+}

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 212301.
jvikstrom marked 14 inline comments as done.
jvikstrom added a comment.

Address comments.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475

Files:
  clang-tools-extra/clangd/ClangdLSPServer.cpp
  clang-tools-extra/clangd/ClangdLSPServer.h
  clang-tools-extra/clangd/ClangdServer.cpp
  clang-tools-extra/clangd/ClangdServer.h
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/semantic-highlighting.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,9 +29,7 @@
   return Tokens;
 }
 
-void checkHighlightings(llvm::StringRef Code) {
-  Annotations Test(Code);
-  auto AST = TestTU::withCode(Test.code()).build();
+std::vector getExpectedTokens(Annotations &Test) {
   static const std::map KindToString{
   {HighlightingKind::Variable, "Variable"},
   {HighlightingKind::Function, "Function"},
@@ -48,10 +46,47 @@
 Test.ranges(KindString.second), KindString.first);
 ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
   }
+  llvm::sort(ExpectedTokens);
+  return ExpectedTokens;
+}
+
+void checkHighlightings(llvm::StringRef Code) {
+  Annotations Test(Code);
+  auto AST = TestTU::withCode(Test.code()).build();
+  std::vector ActualTokens = getSemanticHighlightings(AST);
+  EXPECT_THAT(ActualTokens, getExpectedTokens(Test));
+}
+
+// Any annotations in OldCode and NewCode are converted into their corresponding
+// HighlightingToken. The tokens are diffed against each other. Any lines where
+// the tokens should diff must be marked with a ^ somewhere on that line in
+// NewCode. If there are diffs that aren't marked with ^ the test fails. The
+// test also fails if there are lines marked with ^ that don't differ.
+void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) {
+  Annotations OldTest(OldCode);
+  Annotations NewTest(NewCode);
+  std::vector OldTokens = getExpectedTokens(OldTest);
+  std::vector NewTokens = getExpectedTokens(NewTest);
+
+  llvm::DenseMap> ExpectedLines;
+  for (const Position &Point : NewTest.points()) {
+ExpectedLines[Point.line]; // Default initialize to an empty line. Tokens
+   // are inserted on these lines later.
+  }
+  std::vector ExpectedLinePairHighlighting;
+  for (const HighlightingToken &Token : NewTokens) {
+auto It = ExpectedLines.find(Token.R.start.line);
+if (It != ExpectedLines.end())
+  It->second.push_back(Token);
+  }
+  for (auto &LineTokens : ExpectedLines)
+ExpectedLinePairHighlighting.push_back(
+{LineTokens.first, LineTokens.second});
 
-  auto ActualTokens = getSemanticHighlightings(AST);
-  EXPECT_THAT(ActualTokens, testing::UnorderedElementsAreArray(ExpectedTokens))
-   << "Inputs is:\n" << Code;
+  std::vector ActualDiffed =
+  diffHighlightings(NewTokens, OldTokens, NewCode.count('\n'));
+  EXPECT_THAT(ActualDiffed,
+  testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting));
 }
 
 TEST(SemanticHighlighting, GetsCorrectTokens) {
@@ -226,8 +261,9 @@
 std::atomic Count = {0};
 
 void onDiagnosticsReady(PathRef, std::vector) override {}
-void onHighlightingsReady(
-PathRef File, std::vector Highlightings) override {
+void onHighlightingsReady(PathRef File,
+  std::vector Highlightings,
+  int NLines) override {
   ++Count;
 }
   };
@@ -252,21 +288,124 @@
 return Pos;
   };
 
-  std::vector Tokens{
-  {HighlightingKind::Variable,
-Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
-  {HighlightingKind::Function,
-Range{CreatePosition(3, 4), CreatePosition(3, 7)}},
-  {HighlightingKind::Variable,
-Range{CreatePosition(1, 1), CreatePosition(1, 5)}}};
+  std::vector Tokens{
+  {3,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
+{HighlightingKind::Function,
+ Range{CreatePosition(3, 4), CreatePosition(3, 7),
+  {1,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(1, 1), CreatePosition(1, 5)};
   std::vector ActualResults =
   toSemanticHighlightingInformation(Tokens);
   std::vector ExpectedResults = {
-  {1, "AQAEAAA="},
-  {3, "CAAEAAAEAAMAAQ=="}};
+  {3, "CAAEAAAEAAMAAQ=="}, {1, "AQAEAAA="}};
   EXPECT_EQ(ActualResults, ExpectedResults);
 }
 
+TEST(SemanticHighlighting, HighlightingDiffer) {
+  s

[PATCH] D64922: [clangd] Added option to enable semantic highlighting via an experimental capability

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom abandoned this revision.
jvikstrom added a comment.

In D64922#1603759 , @nridge wrote:

> I believe this is not necessary. You can add text document capabilities in 
> the vscode client extension like this:
>
>   class SemanticHighlightingFeature implements vscodelc.StaticFeature {
> fillClientCapabilities(capabilities: vscodelc.ClientCapabilities): void {
>   const textDocumentCapabilities:
>   vscodelc.TextDocumentClientCapabilities & { 
> semanticHighlightingCapabilities?: { semanticHighlighting: boolean } }
>   = capabilities.textDocument;
>   textDocumentCapabilities.semanticHighlightingCapabilities = { 
> semanticHighlighting: true };
> }
>   }
>  
>   ...
>  
>   clangdClient.registerFeature(new SemanticHighlightingFeature());
>
>
> What this is doing is creating a new type on the fly which extends 
> `vscodelc.TextDocumentCapabilities` with a new optional field 
> `semanticHighlightingCapabilities`, and then casts 
> `capabilities.textDocument` to that type.


That definitely works, I had no idea that was a feature you could use in 
typescript. Thanks

Abandoning in favor of using type intersection in typescript instead.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64922/new/

https://reviews.llvm.org/D64922



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom added inline comments.



Comment at: clang-tools-extra/clangd/SemanticHighlighting.cpp:294
+  // ArrayRefs to the current line in the highlights.
+  ArrayRef NewLine(New.begin(),
+  /*length*/0UL);

ilya-biryukov wrote:
> Could we try to find a better name for this? Having `Line` and `NextLine()` 
> which represent line numbers and `NewLine` which represents highlightings, 
> creates some confusion.
I renamed the `Line` and `NextLine()` instead. Could also rename `NewLine` to 
something like `NewLineHighlightings` but I feel like it almost becomes to 
verbose at that point.



Comment at: clang-tools-extra/clangd/SemanticHighlighting.cpp:309
+  // If the New file has fewer lines than the Old file we don't want to send
+  // highlightings beyond the end of the file. That might crash the client.
+  for (int Line = 0; Line != std::numeric_limits::max() && Line < NLines;

hokein wrote:
> nit: I believe theia is crashing because of this? could we file a bug to 
> theia? I think it would be nice for client to be robust on this case. 
I seem to be misremembering, when I first started testing in theia I think I 
was crashing the client (due to a broken base64 encoding function, will take a 
look and see what was actually happening, can't quite remember)
It actually seems like theia just ignores any out of bounds highlightings.

So this could be removed, it will just sometimes return highlightings that are 
outside of the file and hopefully any future clients would handle that as well. 


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D65395: [clangd] Added a skeleton for a semantic highlighting feature to the vscode extension.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 212302.
jvikstrom added a comment.

Removed the experimental capability and added it to the normal capabilites 
using type intersection.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D65395/new/

https://reviews.llvm.org/D65395

Files:
  clang-tools-extra/clangd/clients/clangd-vscode/.gitignore
  clang-tools-extra/clangd/clients/clangd-vscode/src/SemanticHighlighting.ts
  clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts

Index: clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
===
--- clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
+++ clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
@@ -1,5 +1,6 @@
 import * as vscode from 'vscode';
 import * as vscodelc from 'vscode-languageclient';
+import { SemanticHighlighting } from './SemanticHighlighting';
 
 /**
  * Method to get workspace configuration option
@@ -12,9 +13,9 @@
 }
 
 namespace SwitchSourceHeaderRequest {
-export const type =
-new vscodelc.RequestType('textDocument/switchSourceHeader');
+export const type =
+new vscodelc.RequestType('textDocument/switchSourceHeader');
 }
 
 class FileStatus {
@@ -32,8 +33,8 @@
 const path = vscode.window.activeTextEditor.document.fileName;
 const status = this.statuses.get(path);
 if (!status) {
-  this.statusBarItem.hide();
-  return;
+this.statusBarItem.hide();
+return;
 }
 this.statusBarItem.text = `clangd: ` + status.state;
 this.statusBarItem.show();
@@ -73,25 +74,30 @@
 // However, VSCode does not have CUDA as a supported language yet, so we
 // cannot add a corresponding activationEvent for CUDA files and clangd will
 // *not* load itself automatically on '.cu' files.
-const cudaFilePattern: string = '**/*.{' +['cu'].join()+ '}';
+const cudaFilePattern: string = '**/*.{' + ['cu'].join() + '}';
 const clientOptions: vscodelc.LanguageClientOptions = {
 // Register the server for c-family and cuda files.
 documentSelector: [
 { scheme: 'file', language: 'c' },
 { scheme: 'file', language: 'cpp' },
-{ scheme: 'file', language: 'objective-c'},
-{ scheme: 'file', language: 'objective-cpp'},
+{ scheme: 'file', language: 'objective-c' },
+{ scheme: 'file', language: 'objective-cpp' },
 { scheme: 'file', pattern: cudaFilePattern },
 ],
 synchronize: !syncFileEvents ? undefined : {
-// FIXME: send sync file events when clangd provides implemenatations.
+// FIXME: send sync file events when clangd provides implemenatations.
 },
 initializationOptions: { clangdFileStatus: true },
 // Do not switch to output window when clangd returns output
 revealOutputChannelOn: vscodelc.RevealOutputChannelOn.Never
 };
 
-const clangdClient = new vscodelc.LanguageClient('Clang Language Server',serverOptions, clientOptions);
+const clangdClient = new vscodelc.LanguageClient('Clang Language Server', serverOptions, clientOptions);
+const semanticHighlightingFeature = new SemanticHighlighting.Feature();
+clangdClient.registerFeature(semanticHighlightingFeature);
+// The notification handler must be registered after the client is ready or the client will crash.
+clangdClient.onReady().then(() => clangdClient.onNotification(SemanticHighlighting.NotificationType, semanticHighlightingFeature.handleNotification));
+
 console.log('Clang Language Server is now active!');
 context.subscriptions.push(clangdClient.start());
 context.subscriptions.push(vscode.commands.registerCommand(
@@ -131,5 +137,5 @@
 // An empty place holder for the activate command, otherwise we'll get an
 // "command is not registered" error.
 context.subscriptions.push(vscode.commands.registerCommand(
-'clangd-vscode.activate', async () => {}));
+'clangd-vscode.activate', async () => { }));
 }
Index: clang-tools-extra/clangd/clients/clangd-vscode/src/SemanticHighlighting.ts
===
--- /dev/null
+++ clang-tools-extra/clangd/clients/clangd-vscode/src/SemanticHighlighting.ts
@@ -0,0 +1,34 @@
+import * as vscodelc from 'vscode-languageclient';
+import { DocumentSelector } from 'vscode-languageclient';
+
+export namespace SemanticHighlighting {
+interface HighlightingInformation {
+textDocument: {
+uri: String,
+},
+lines: [{
+line: number,
+tokens: string,
+}],
+}
+
+export const NotificationType = new vscodelc.NotificationType<{}, void>('textDocument/semanticHighlighting');
+
+// The feature that should be registered in the vscode lsp for enabling experimental semant

[PATCH] D65443: [clangd] Fix a regression in rL366996.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom accepted this revision.
jvikstrom added inline comments.
This revision is now accepted and ready to land.



Comment at: 
clang-tools-extra/clangd/refactor/tweaks/AnnotateHighlightings.cpp:43
+  if (!CommonDecl) {
+// Now we hit the TUDecl case where commonAncestor() returns null intently.
+// We only annotate tokens in the main file, so use the default traversal

Shouldn't `intently` -> `instantly`?


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D65443/new/

https://reviews.llvm.org/D65443



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D65395: [clangd] Added a skeleton for a semantic highlighting feature to the vscode extension.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 212523.
jvikstrom added a comment.

Added check that clangd server actually supports semantic highlighting.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D65395/new/

https://reviews.llvm.org/D65395

Files:
  clang-tools-extra/clangd/clients/clangd-vscode/.gitignore
  clang-tools-extra/clangd/clients/clangd-vscode/src/SemanticHighlighting.ts
  clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts

Index: clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
===
--- clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
+++ clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
@@ -1,5 +1,6 @@
 import * as vscode from 'vscode';
 import * as vscodelc from 'vscode-languageclient';
+import { SemanticHighlighting } from './SemanticHighlighting';
 
 /**
  * Method to get workspace configuration option
@@ -12,9 +13,9 @@
 }
 
 namespace SwitchSourceHeaderRequest {
-export const type =
-new vscodelc.RequestType('textDocument/switchSourceHeader');
+export const type =
+new vscodelc.RequestType('textDocument/switchSourceHeader');
 }
 
 class FileStatus {
@@ -32,8 +33,8 @@
 const path = vscode.window.activeTextEditor.document.fileName;
 const status = this.statuses.get(path);
 if (!status) {
-  this.statusBarItem.hide();
-  return;
+this.statusBarItem.hide();
+return;
 }
 this.statusBarItem.text = `clangd: ` + status.state;
 this.statusBarItem.show();
@@ -73,25 +74,30 @@
 // However, VSCode does not have CUDA as a supported language yet, so we
 // cannot add a corresponding activationEvent for CUDA files and clangd will
 // *not* load itself automatically on '.cu' files.
-const cudaFilePattern: string = '**/*.{' +['cu'].join()+ '}';
+const cudaFilePattern: string = '**/*.{' + ['cu'].join() + '}';
 const clientOptions: vscodelc.LanguageClientOptions = {
 // Register the server for c-family and cuda files.
 documentSelector: [
 { scheme: 'file', language: 'c' },
 { scheme: 'file', language: 'cpp' },
-{ scheme: 'file', language: 'objective-c'},
-{ scheme: 'file', language: 'objective-cpp'},
+{ scheme: 'file', language: 'objective-c' },
+{ scheme: 'file', language: 'objective-cpp' },
 { scheme: 'file', pattern: cudaFilePattern },
 ],
 synchronize: !syncFileEvents ? undefined : {
-// FIXME: send sync file events when clangd provides implemenatations.
+// FIXME: send sync file events when clangd provides implemenatations.
 },
 initializationOptions: { clangdFileStatus: true },
 // Do not switch to output window when clangd returns output
 revealOutputChannelOn: vscodelc.RevealOutputChannelOn.Never
 };
 
-const clangdClient = new vscodelc.LanguageClient('Clang Language Server',serverOptions, clientOptions);
+const clangdClient = new vscodelc.LanguageClient('Clang Language Server', serverOptions, clientOptions);
+const semanticHighlightingFeature = new SemanticHighlighting.Feature();
+clangdClient.registerFeature(semanticHighlightingFeature);
+// The notification handler must be registered after the client is ready or the client will crash.
+clangdClient.onReady().then(() => clangdClient.onNotification(SemanticHighlighting.NotificationType, semanticHighlightingFeature.handleNotification));
+
 console.log('Clang Language Server is now active!');
 context.subscriptions.push(clangdClient.start());
 context.subscriptions.push(vscode.commands.registerCommand(
@@ -131,5 +137,5 @@
 // An empty place holder for the activate command, otherwise we'll get an
 // "command is not registered" error.
 context.subscriptions.push(vscode.commands.registerCommand(
-'clangd-vscode.activate', async () => {}));
+'clangd-vscode.activate', async () => { }));
 }
Index: clang-tools-extra/clangd/clients/clangd-vscode/src/SemanticHighlighting.ts
===
--- /dev/null
+++ clang-tools-extra/clangd/clients/clangd-vscode/src/SemanticHighlighting.ts
@@ -0,0 +1,36 @@
+import * as vscodelc from 'vscode-languageclient';
+import { DocumentSelector } from 'vscode-languageclient';
+
+export namespace SemanticHighlighting {
+interface HighlightingInformation {
+textDocument: {
+uri: String,
+},
+lines: [{
+line: number,
+tokens: string,
+}],
+}
+
+export const NotificationType = new vscodelc.NotificationType<{}, void>('textDocument/semanticHighlighting');
+
+// The feature that should be registered in the vscode lsp for enabling experimental semantic highlighting.
+export 

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 212522.
jvikstrom marked 2 inline comments as done.
jvikstrom added a comment.

Stopped using expensive getLineNumber function.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475

Files:
  clang-tools-extra/clangd/ClangdLSPServer.cpp
  clang-tools-extra/clangd/ClangdLSPServer.h
  clang-tools-extra/clangd/ClangdServer.cpp
  clang-tools-extra/clangd/ClangdServer.h
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/semantic-highlighting.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,9 +29,7 @@
   return Tokens;
 }
 
-void checkHighlightings(llvm::StringRef Code) {
-  Annotations Test(Code);
-  auto AST = TestTU::withCode(Test.code()).build();
+std::vector getExpectedTokens(Annotations &Test) {
   static const std::map KindToString{
   {HighlightingKind::Variable, "Variable"},
   {HighlightingKind::Function, "Function"},
@@ -48,10 +46,47 @@
 Test.ranges(KindString.second), KindString.first);
 ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
   }
+  llvm::sort(ExpectedTokens);
+  return ExpectedTokens;
+}
+
+void checkHighlightings(llvm::StringRef Code) {
+  Annotations Test(Code);
+  auto AST = TestTU::withCode(Test.code()).build();
+  std::vector ActualTokens = getSemanticHighlightings(AST);
+  EXPECT_THAT(ActualTokens, getExpectedTokens(Test));
+}
+
+// Any annotations in OldCode and NewCode are converted into their corresponding
+// HighlightingToken. The tokens are diffed against each other. Any lines where
+// the tokens should diff must be marked with a ^ somewhere on that line in
+// NewCode. If there are diffs that aren't marked with ^ the test fails. The
+// test also fails if there are lines marked with ^ that don't differ.
+void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) {
+  Annotations OldTest(OldCode);
+  Annotations NewTest(NewCode);
+  std::vector OldTokens = getExpectedTokens(OldTest);
+  std::vector NewTokens = getExpectedTokens(NewTest);
+
+  llvm::DenseMap> ExpectedLines;
+  for (const Position &Point : NewTest.points()) {
+ExpectedLines[Point.line]; // Default initialize to an empty line. Tokens
+   // are inserted on these lines later.
+  }
+  std::vector ExpectedLinePairHighlighting;
+  for (const HighlightingToken &Token : NewTokens) {
+auto It = ExpectedLines.find(Token.R.start.line);
+if (It != ExpectedLines.end())
+  It->second.push_back(Token);
+  }
+  for (auto &LineTokens : ExpectedLines)
+ExpectedLinePairHighlighting.push_back(
+{LineTokens.first, LineTokens.second});
 
-  auto ActualTokens = getSemanticHighlightings(AST);
-  EXPECT_THAT(ActualTokens, testing::UnorderedElementsAreArray(ExpectedTokens))
-   << "Inputs is:\n" << Code;
+  std::vector ActualDiffed =
+  diffHighlightings(NewTokens, OldTokens, NewCode.count('\n'));
+  EXPECT_THAT(ActualDiffed,
+  testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting));
 }
 
 TEST(SemanticHighlighting, GetsCorrectTokens) {
@@ -226,8 +261,9 @@
 std::atomic Count = {0};
 
 void onDiagnosticsReady(PathRef, std::vector) override {}
-void onHighlightingsReady(
-PathRef File, std::vector Highlightings) override {
+void onHighlightingsReady(PathRef File,
+  std::vector Highlightings,
+  int NLines) override {
   ++Count;
 }
   };
@@ -252,21 +288,124 @@
 return Pos;
   };
 
-  std::vector Tokens{
-  {HighlightingKind::Variable,
-Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
-  {HighlightingKind::Function,
-Range{CreatePosition(3, 4), CreatePosition(3, 7)}},
-  {HighlightingKind::Variable,
-Range{CreatePosition(1, 1), CreatePosition(1, 5)}}};
+  std::vector Tokens{
+  {3,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
+{HighlightingKind::Function,
+ Range{CreatePosition(3, 4), CreatePosition(3, 7),
+  {1,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(1, 1), CreatePosition(1, 5)};
   std::vector ActualResults =
   toSemanticHighlightingInformation(Tokens);
   std::vector ExpectedResults = {
-  {1, "AQAEAAA="},
-  {3, "CAAEAAAEAAMAAQ=="}};
+  {3, "CAAEAAAEAAMAAQ=="}, {1, "AQAEAAA="}};
   EXPECT_EQ(ActualResults, ExpectedResults);
 }
 
+TEST(SemanticHighlightin

[PATCH] D65395: [clangd] Added a skeleton for a semantic highlighting feature to the vscode extension.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom added a comment.

In D65395#1607889 , @hokein wrote:

> The skeleton looks good, but apparently it does nothing (only enable the 
> feature), do you have a small working prototype?


I've got a prototype that colors all the highlightings red, but it would make 
the CL become quite big if I added that as well. This felt like a reasonable 
place to divide it. Do you want me to add that as well or should I keep it as 
2-3 CLs? (i.e. this -> TM scopes -> highlighting rendering)


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D65395/new/

https://reviews.llvm.org/D65395



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 212528.
jvikstrom marked an inline comment as done.
jvikstrom added a comment.

Copy outside lock.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475

Files:
  clang-tools-extra/clangd/ClangdLSPServer.cpp
  clang-tools-extra/clangd/ClangdLSPServer.h
  clang-tools-extra/clangd/ClangdServer.cpp
  clang-tools-extra/clangd/ClangdServer.h
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/semantic-highlighting.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,9 +29,7 @@
   return Tokens;
 }
 
-void checkHighlightings(llvm::StringRef Code) {
-  Annotations Test(Code);
-  auto AST = TestTU::withCode(Test.code()).build();
+std::vector getExpectedTokens(Annotations &Test) {
   static const std::map KindToString{
   {HighlightingKind::Variable, "Variable"},
   {HighlightingKind::Function, "Function"},
@@ -48,10 +46,47 @@
 Test.ranges(KindString.second), KindString.first);
 ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
   }
+  llvm::sort(ExpectedTokens);
+  return ExpectedTokens;
+}
+
+void checkHighlightings(llvm::StringRef Code) {
+  Annotations Test(Code);
+  auto AST = TestTU::withCode(Test.code()).build();
+  std::vector ActualTokens = getSemanticHighlightings(AST);
+  EXPECT_THAT(ActualTokens, getExpectedTokens(Test));
+}
+
+// Any annotations in OldCode and NewCode are converted into their corresponding
+// HighlightingToken. The tokens are diffed against each other. Any lines where
+// the tokens should diff must be marked with a ^ somewhere on that line in
+// NewCode. If there are diffs that aren't marked with ^ the test fails. The
+// test also fails if there are lines marked with ^ that don't differ.
+void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) {
+  Annotations OldTest(OldCode);
+  Annotations NewTest(NewCode);
+  std::vector OldTokens = getExpectedTokens(OldTest);
+  std::vector NewTokens = getExpectedTokens(NewTest);
+
+  llvm::DenseMap> ExpectedLines;
+  for (const Position &Point : NewTest.points()) {
+ExpectedLines[Point.line]; // Default initialize to an empty line. Tokens
+   // are inserted on these lines later.
+  }
+  std::vector ExpectedLinePairHighlighting;
+  for (const HighlightingToken &Token : NewTokens) {
+auto It = ExpectedLines.find(Token.R.start.line);
+if (It != ExpectedLines.end())
+  It->second.push_back(Token);
+  }
+  for (auto &LineTokens : ExpectedLines)
+ExpectedLinePairHighlighting.push_back(
+{LineTokens.first, LineTokens.second});
 
-  auto ActualTokens = getSemanticHighlightings(AST);
-  EXPECT_THAT(ActualTokens, testing::UnorderedElementsAreArray(ExpectedTokens))
-   << "Inputs is:\n" << Code;
+  std::vector ActualDiffed =
+  diffHighlightings(NewTokens, OldTokens, NewCode.count('\n'));
+  EXPECT_THAT(ActualDiffed,
+  testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting));
 }
 
 TEST(SemanticHighlighting, GetsCorrectTokens) {
@@ -226,8 +261,9 @@
 std::atomic Count = {0};
 
 void onDiagnosticsReady(PathRef, std::vector) override {}
-void onHighlightingsReady(
-PathRef File, std::vector Highlightings) override {
+void onHighlightingsReady(PathRef File,
+  std::vector Highlightings,
+  int NLines) override {
   ++Count;
 }
   };
@@ -252,21 +288,124 @@
 return Pos;
   };
 
-  std::vector Tokens{
-  {HighlightingKind::Variable,
-Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
-  {HighlightingKind::Function,
-Range{CreatePosition(3, 4), CreatePosition(3, 7)}},
-  {HighlightingKind::Variable,
-Range{CreatePosition(1, 1), CreatePosition(1, 5)}}};
+  std::vector Tokens{
+  {3,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
+{HighlightingKind::Function,
+ Range{CreatePosition(3, 4), CreatePosition(3, 7),
+  {1,
+   {{HighlightingKind::Variable,
+ Range{CreatePosition(1, 1), CreatePosition(1, 5)};
   std::vector ActualResults =
   toSemanticHighlightingInformation(Tokens);
   std::vector ExpectedResults = {
-  {1, "AQAEAAA="},
-  {3, "CAAEAAAEAAMAAQ=="}};
+  {3, "CAAEAAAEAAMAAQ=="}, {1, "AQAEAAA="}};
   EXPECT_EQ(ActualResults, ExpectedResults);
 }
 
+TEST(SemanticHighlighting, HighlightingDiffer) {
+  s

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom added inline comments.



Comment at: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp:408
+}
+
 } // namespace

ilya-biryukov wrote:
> ilya-biryukov wrote:
> > Could you also add a separate test that checks diffs when the number of 
> > lines is getting smaller (i.e. taking `NLines` into account)
> I would expect this to be better handled outside `checkDiffedHighlights` to 
> avoid over-generalizing this function. But up to you.
That's what this test does: 

```
{
R"(
$Class[[A]]
$Variable[[A]]
$Class[[A]]
$Variable[[A]]
  )",
R"(
$Class[[A]]
$Variable[[A]]
  )"},
```

But do you mean I should do a testcase like:



```
{
R"(
$Class[[A]]
$Variable[[A]]
$Class[[A]]
$Variable[[A]]
  )",
R"(
$Class[[A]]
$Variable[[A]]
$Class[[A]]
$Variable[[A]]
  )"},
```
And just set NLines = 2 instead when doing the diffing?



Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D65510: [clangd] Fix implicit template instatiations appearing as topLevelDecls.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom created this revision.
jvikstrom added reviewers: hokein, ilya-biryukov.
Herald added subscribers: cfe-commits, kadircet, arphaman, jkorous, MaskRay.
Herald added a project: clang.

The parser gives implicit template instantiations to the action's 
HandleTopLevelDecls callback. This makes semantic highlighting highlight these 
templated functions multiple times. Fixed by filtering on if a Decl is an 
implicit function/variable/class instantiation. Also added a testcase to 
semantic highlighting on this.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D65510

Files:
  clang-tools-extra/clangd/ClangdUnit.cpp
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp


Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -214,7 +214,25 @@
 
   template
   void $Function[[foo]]($TemplateParameter[[T]] ...);
-)cpp"};
+)cpp",
+R"cpp(
+  class $Class[[Foo]] {
+  public:
+template 
+void $Method[[ff]]($TemplateParameter[[T1]]) {}
+  };
+  template 
+  void $Function[[ff]]($TemplateParameter[[T1]]) {}
+  enum $Enum[[E]] { $EnumConstant[[VALUE]] };
+  void $Function[[s]]() {
+$Function[[ff]]($EnumConstant[[VALUE]]);
+$Function[[ff]]($Class[[Foo]]());
+$Class[[Foo]] $Variable[[s]];
+$Variable[[s]].$Method[[ff]]($EnumConstant[[VALUE]]);
+$Variable[[s]].$Method[[ff]]($Class[[Foo]]());
+  }
+)cpp"
+  };
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
   }
Index: clang-tools-extra/clangd/ClangdUnit.cpp
===
--- clang-tools-extra/clangd/ClangdUnit.cpp
+++ clang-tools-extra/clangd/ClangdUnit.cpp
@@ -19,8 +19,11 @@
 #include "index/CanonicalIncludes.h"
 #include "index/Index.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclCXX.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceManager.h"
+#include "clang/Basic/Specifiers.h"
 #include "clang/Basic/TokenKinds.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/CompilerInvocation.h"
@@ -60,6 +63,12 @@
   return Vec.capacity() * sizeof(T);
 }
 
+template  bool isImplicitTemplateInstantiation(const Decl *D) {
+  if (const auto *TD = dyn_cast(D))
+return TD->getTemplateSpecializationKind() == TSK_ImplicitInstantiation;
+  return false;
+}
+
 class DeclTrackingASTConsumer : public ASTConsumer {
 public:
   DeclTrackingASTConsumer(std::vector &TopLevelDecls)
@@ -70,6 +79,10 @@
   auto &SM = D->getASTContext().getSourceManager();
   if (!isInsideMainFile(D->getLocation(), SM))
 continue;
+  if (isImplicitTemplateInstantiation(D) ||
+  isImplicitTemplateInstantiation(D) ||
+  isImplicitTemplateInstantiation(D))
+continue;
 
   // ObjCMethodDecl are not actually top-level decls.
   if (isa(D))


Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -214,7 +214,25 @@
 
   template
   void $Function[[foo]]($TemplateParameter[[T]] ...);
-)cpp"};
+)cpp",
+R"cpp(
+  class $Class[[Foo]] {
+  public:
+template 
+void $Method[[ff]]($TemplateParameter[[T1]]) {}
+  };
+  template 
+  void $Function[[ff]]($TemplateParameter[[T1]]) {}
+  enum $Enum[[E]] { $EnumConstant[[VALUE]] };
+  void $Function[[s]]() {
+$Function[[ff]]($EnumConstant[[VALUE]]);
+$Function[[ff]]($Class[[Foo]]());
+$Class[[Foo]] $Variable[[s]];
+$Variable[[s]].$Method[[ff]]($EnumConstant[[VALUE]]);
+$Variable[[s]].$Method[[ff]]($Class[[Foo]]());
+  }
+)cpp"
+  };
   for (const auto &TestCase : TestCases) {
 checkHighlightings(TestCase);
   }
Index: clang-tools-extra/clangd/ClangdUnit.cpp
===
--- clang-tools-extra/clangd/ClangdUnit.cpp
+++ clang-tools-extra/clangd/ClangdUnit.cpp
@@ -19,8 +19,11 @@
 #include "index/CanonicalIncludes.h"
 #include "index/Index.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclCXX.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceManager.h"
+#include "clang/Basic/Specifiers.h"
 #include "clang/Basic/TokenKinds.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/CompilerInvocation.h"
@@ -60,6 +63,12 @@
   return Vec.capacity() * sizeof(T);
 }
 
+template  bool isImplicitTemplateInstantiation(const Decl *D) {
+  if (con

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom marked an inline comment as done.
jvikstrom added inline comments.



Comment at: clang-tools-extra/clangd/ClangdLSPServer.cpp:1125
+Old = std::move(FileToHighlightings[File]);
+FileToHighlightings[File] = Highlightings;
+  }

hokein wrote:
> ilya-biryukov wrote:
> > NIT: avoid copying (from `Highlightings` into the map) under a lock, make 
> > the copy outside the lock instead
> I think we can even avoid copy, since we only use the `Highlightings` for 
> calculating the diff.
> 
> Maybe just 
> 
> ```
> {
> std::lock_guard Lock(HighlightingsMutex);
> Old = std::move(FileToHighlightings[File]);
> }
> // calculate the diff.
> {  
>  std::lock_guard Lock(HighlightingsMutex);
>  FileToHighlightings[File] = std::move(Highlightings);
> }
> ```
I think I changed this to only lock once (and copy instead) at the same time me 
and @ilya-biryukov were talking about the race condition (?)


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D64475: [clangd] Duplicate lines of semantic highlightings sent removed.

[Johan Vikström via Phabricator via cfe-commits]

This revision was automatically updated to reflect the committed changes.
Closed by commit rL367521: [clangd] Duplicate lines of semantic highlightings 
sent removed. (authored by jvikstrom, committed by ).
Herald added a project: LLVM.
Herald added a subscriber: llvm-commits.

Changed prior to commit:
  https://reviews.llvm.org/D64475?vs=212528&id=212747#toc

Repository:
  rL LLVM

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D64475/new/

https://reviews.llvm.org/D64475

Files:
  clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp
  clang-tools-extra/trunk/clangd/ClangdLSPServer.h
  clang-tools-extra/trunk/clangd/ClangdServer.cpp
  clang-tools-extra/trunk/clangd/ClangdServer.h
  clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp
  clang-tools-extra/trunk/clangd/SemanticHighlighting.h
  clang-tools-extra/trunk/clangd/test/semantic-highlighting.test
  clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/trunk/clangd/test/semantic-highlighting.test
===
--- clang-tools-extra/trunk/clangd/test/semantic-highlighting.test
+++ clang-tools-extra/trunk/clangd/test/semantic-highlighting.test
@@ -49,6 +49,50 @@
 # CHECK-NEXT:  }
 # CHECK-NEXT:}
 ---
+{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo2.cpp","languageId":"cpp","version":1,"text":"int x = 2;\nint y = 2;"}}}
+#  CHECK:  "method": "textDocument/semanticHighlighting",
+# CHECK-NEXT:  "params": {
+# CHECK-NEXT:"lines": [
+# CHECK-NEXT:  {
+# CHECK-NEXT:"line": 0,
+# CHECK-NEXT:"tokens": "BAABAAA="
+# CHECK-NEXT:  }
+# CHECK-NEXT:  {
+# CHECK-NEXT:"line": 1,
+# CHECK-NEXT:"tokens": "BAABAAA="
+# CHECK-NEXT:  }
+# CHECK-NEXT:],
+# CHECK-NEXT:"textDocument": {
+# CHECK-NEXT:  "uri": "file:///clangd-test/foo2.cpp"
+# CHECK-NEXT:}
+# CHECK-NEXT:  }
+# CHECK-NEXT:}
+---
+{"jsonrpc":"2.0","method":"textDocument/didChange","params":{"textDocument":{"uri":"test:///foo.cpp","version":2},"contentChanges": [{"range":{"start": {"line": 0,"character": 10},"end": {"line": 0,"character": 10}},"rangeLength": 0,"text": "\nint y = 2;"}]}}
+#  CHECK:  "method": "textDocument/semanticHighlighting",
+# CHECK-NEXT:  "params": {
+# CHECK-NEXT:"lines": [
+# CHECK-NEXT:  {
+# CHECK-NEXT:"line": 1,
+# CHECK-NEXT:"tokens": "BAABAAA="
+# CHECK-NEXT:  }
+# CHECK-NEXT:   ],
+# CHECK-NEXT:"textDocument": {
+# CHECK-NEXT:  "uri": "file:///clangd-test/foo.cpp"
+# CHECK-NEXT:}
+# CHECK-NEXT:  }
+# CHECK-NEXT:}
+---
+{"jsonrpc":"2.0","method":"textDocument/didChange","params":{"textDocument":{"uri":"test:///foo.cpp","version":2},"contentChanges": [{"range":{"start": {"line": 0,"character": 10},"end": {"line": 1,"character": 10}},"rangeLength": 11,"text": ""}]}}
+#  CHECK:  "method": "textDocument/semanticHighlighting",
+# CHECK-NEXT:  "params": {
+# CHECK-NEXT:"lines": [],
+# CHECK-NEXT:"textDocument": {
+# CHECK-NEXT:  "uri": "file:///clangd-test/foo.cpp"
+# CHECK-NEXT:}
+# CHECK-NEXT:  }
+# CHECK-NEXT:}
+---
 {"jsonrpc":"2.0","id":3,"method":"shutdown"}
 ---
 {"jsonrpc":"2.0","method":"exit"}
Index: clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,9 +29,7 @@
   return Tokens;
 }
 
-void checkHighlightings(llvm::StringRef Code) {
-  Annotations Test(Code);
-  auto AST = TestTU::withCode(Test.code()).build();
+std::vector getExpectedTokens(Annotations &Test) {
   static const std::map KindToString{
   {HighlightingKind::Variable, "Variable"},
   {HighlightingKind::Function, "Function"},
@@ -48,10 +46,47 @@
 Test.ranges(KindString.second), KindString.first);
 ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
   }
+  llvm::sort(ExpectedTokens);
+  return ExpectedTokens;
+}
+
+void checkHighlightings(llvm::StringRef Code) {
+  Annotations Test(Code);
+  auto AST = TestTU::withCode(Test.code()).build();
+  std::vector ActualTokens = getSemanticHighlightings(AST);
+  EXPECT_THAT(ActualTokens, getExpectedTokens(Test));
+}
 
-  auto ActualTokens = getSemanticHighlightings(AST);
-  EXPECT_THAT(ActualTokens, testing::UnorderedElementsAreArray(ExpectedTokens))
-   << "Inputs is:\n" << Code;
+// Any annotations in OldCode and NewCode are converted into their corresponding
+// HighlightingToken. The tokens are diffed against each other. Any lines where
+// the tokens should diff must be marked with a ^ somewhere on that line in
+// NewCode. If there are diffs that aren't marked with ^ the test fails. The
+// test also fails if there are lines marked with ^ that don't differ.
+void checkDiffedHighlights(llvm::StringRe

[PATCH] D65637: [clangd] [WIP] Semantic highlighting prototype for the vscode extension.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom created this revision.
jvikstrom added reviewers: hokein, sammccall, ilya-biryukov.
Herald added subscribers: cfe-commits, kadircet, arphaman, jkorous, MaskRay.
Herald added a project: clang.

Contains all the functionality for the highlighting, is going to get cleaned up 
a lot though. Might end up wanting to create a color theme as well, none of the 
default ones are that great with this...


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D65637

Files:
  clang-tools-extra/clangd/clients/clangd-vscode/.gitignore
  clang-tools-extra/clangd/clients/clangd-vscode/src/SemanticHighlighting.ts
  clang-tools-extra/clangd/clients/clangd-vscode/src/TextMate.ts
  clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
  clang-tools-extra/clangd/clients/clangd-vscode/test/extension.test.ts

Index: clang-tools-extra/clangd/clients/clangd-vscode/test/extension.test.ts
===
--- clang-tools-extra/clangd/clients/clangd-vscode/test/extension.test.ts
+++ clang-tools-extra/clangd/clients/clangd-vscode/test/extension.test.ts
@@ -2,14 +2,30 @@
 import * as assert from 'assert';
 
 import * as vscode from 'vscode';
-import * as myExtension from '../src/extension';
+import {SemanticHighlighting} from '../src/SemanticHighlighting';
 
-// TODO: add tests
 suite("Extension Tests", () => {
-
-// Defines a Mocha unit test
-test("Something 1", () => {
-assert.equal(-1, [1, 2, 3].indexOf(5));
-assert.equal(-1, [1, 2, 3].indexOf(0));
+test('HighlightingCache caches incrementaly', () => {
+const feature = new SemanticHighlighting.Feature();
+const uri = 'file://text';
+feature.handleNotification({
+textDocument: {uri: uri},
+lines: [{
+line: 1,
+tokens: 'BAABAAA=',
+}],
+});
+feature.handleNotification({
+textDocument: {uri: uri},
+lines: [{
+line: 2,
+tokens: 'BAABAAA=',
+}],
+});
+//const tokens = feature.tokenCache.files.get(vscode.Uri.parse(uri).toString()).tokens;
+//assert.deepEqual(tokens.get(1), [{character: 4, length: 1, scope: 0}]);
+//assert.deepEqual(tokens.get(2), [{character: 4, length: 1, scope: 0}]);
+
+
 });
 });
\ No newline at end of file
Index: clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
===
--- clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
+++ clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
@@ -1,5 +1,6 @@
 import * as vscode from 'vscode';
 import * as vscodelc from 'vscode-languageclient';
+import { SemanticHighlighting } from './SemanticHighlighting';
 
 /**
  * Method to get workspace configuration option
@@ -12,9 +13,9 @@
 }
 
 namespace SwitchSourceHeaderRequest {
-export const type =
-new vscodelc.RequestType('textDocument/switchSourceHeader');
+export const type =
+new vscodelc.RequestType('textDocument/switchSourceHeader');
 }
 
 class FileStatus {
@@ -32,8 +33,8 @@
 const path = vscode.window.activeTextEditor.document.fileName;
 const status = this.statuses.get(path);
 if (!status) {
-  this.statusBarItem.hide();
-  return;
+this.statusBarItem.hide();
+return;
 }
 this.statusBarItem.text = `clangd: ` + status.state;
 this.statusBarItem.show();
@@ -73,25 +74,30 @@
 // However, VSCode does not have CUDA as a supported language yet, so we
 // cannot add a corresponding activationEvent for CUDA files and clangd will
 // *not* load itself automatically on '.cu' files.
-const cudaFilePattern: string = '**/*.{' +['cu'].join()+ '}';
+const cudaFilePattern: string = '**/*.{' + ['cu'].join() + '}';
 const clientOptions: vscodelc.LanguageClientOptions = {
 // Register the server for c-family and cuda files.
 documentSelector: [
 { scheme: 'file', language: 'c' },
 { scheme: 'file', language: 'cpp' },
-{ scheme: 'file', language: 'objective-c'},
-{ scheme: 'file', language: 'objective-cpp'},
+{ scheme: 'file', language: 'objective-c' },
+{ scheme: 'file', language: 'objective-cpp' },
 { scheme: 'file', pattern: cudaFilePattern },
 ],
 synchronize: !syncFileEvents ? undefined : {
-// FIXME: send sync file events when clangd provides implemenatations.
+// FIXME: send sync file events when clangd provides implemenatations.
 },
 initializationOptions: { clangdFileStatus: true },
 // Do not switch to output window when clangd returns output
 revealOutputChannelOn: vscodelc.RevealOutputChannelOn.Never
 };
 
-const clangdClient = new vscodelc.LanguageClie

[PATCH] D65637: [clangd] [WIP] Semantic highlighting prototype for the vscode extension.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 213011.
jvikstrom added a comment.

Readded guard against rules without scopes that somehow disappeared.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D65637/new/

https://reviews.llvm.org/D65637

Files:
  clang-tools-extra/clangd/clients/clangd-vscode/.gitignore
  clang-tools-extra/clangd/clients/clangd-vscode/src/SemanticHighlighting.ts
  clang-tools-extra/clangd/clients/clangd-vscode/src/TextMate.ts
  clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
  clang-tools-extra/clangd/clients/clangd-vscode/test/extension.test.ts

Index: clang-tools-extra/clangd/clients/clangd-vscode/test/extension.test.ts
===
--- clang-tools-extra/clangd/clients/clangd-vscode/test/extension.test.ts
+++ clang-tools-extra/clangd/clients/clangd-vscode/test/extension.test.ts
@@ -2,14 +2,30 @@
 import * as assert from 'assert';
 
 import * as vscode from 'vscode';
-import * as myExtension from '../src/extension';
+import {SemanticHighlighting} from '../src/SemanticHighlighting';
 
-// TODO: add tests
 suite("Extension Tests", () => {
-
-// Defines a Mocha unit test
-test("Something 1", () => {
-assert.equal(-1, [1, 2, 3].indexOf(5));
-assert.equal(-1, [1, 2, 3].indexOf(0));
+test('HighlightingCache caches incrementaly', () => {
+const feature = new SemanticHighlighting.Feature();
+const uri = 'file://text';
+feature.handleNotification({
+textDocument: {uri: uri},
+lines: [{
+line: 1,
+tokens: 'BAABAAA=',
+}],
+});
+feature.handleNotification({
+textDocument: {uri: uri},
+lines: [{
+line: 2,
+tokens: 'BAABAAA=',
+}],
+});
+//const tokens = feature.tokenCache.files.get(vscode.Uri.parse(uri).toString()).tokens;
+//assert.deepEqual(tokens.get(1), [{character: 4, length: 1, scope: 0}]);
+//assert.deepEqual(tokens.get(2), [{character: 4, length: 1, scope: 0}]);
+
+
 });
 });
\ No newline at end of file
Index: clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
===
--- clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
+++ clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
@@ -1,5 +1,6 @@
 import * as vscode from 'vscode';
 import * as vscodelc from 'vscode-languageclient';
+import { SemanticHighlighting } from './SemanticHighlighting';
 
 /**
  * Method to get workspace configuration option
@@ -12,9 +13,9 @@
 }
 
 namespace SwitchSourceHeaderRequest {
-export const type =
-new vscodelc.RequestType('textDocument/switchSourceHeader');
+export const type =
+new vscodelc.RequestType('textDocument/switchSourceHeader');
 }
 
 class FileStatus {
@@ -32,8 +33,8 @@
 const path = vscode.window.activeTextEditor.document.fileName;
 const status = this.statuses.get(path);
 if (!status) {
-  this.statusBarItem.hide();
-  return;
+this.statusBarItem.hide();
+return;
 }
 this.statusBarItem.text = `clangd: ` + status.state;
 this.statusBarItem.show();
@@ -73,25 +74,30 @@
 // However, VSCode does not have CUDA as a supported language yet, so we
 // cannot add a corresponding activationEvent for CUDA files and clangd will
 // *not* load itself automatically on '.cu' files.
-const cudaFilePattern: string = '**/*.{' +['cu'].join()+ '}';
+const cudaFilePattern: string = '**/*.{' + ['cu'].join() + '}';
 const clientOptions: vscodelc.LanguageClientOptions = {
 // Register the server for c-family and cuda files.
 documentSelector: [
 { scheme: 'file', language: 'c' },
 { scheme: 'file', language: 'cpp' },
-{ scheme: 'file', language: 'objective-c'},
-{ scheme: 'file', language: 'objective-cpp'},
+{ scheme: 'file', language: 'objective-c' },
+{ scheme: 'file', language: 'objective-cpp' },
 { scheme: 'file', pattern: cudaFilePattern },
 ],
 synchronize: !syncFileEvents ? undefined : {
-// FIXME: send sync file events when clangd provides implemenatations.
+// FIXME: send sync file events when clangd provides implemenatations.
 },
 initializationOptions: { clangdFileStatus: true },
 // Do not switch to output window when clangd returns output
 revealOutputChannelOn: vscodelc.RevealOutputChannelOn.Never
 };
 
-const clangdClient = new vscodelc.LanguageClient('Clang Language Server',serverOptions, clientOptions);
+const clangdClient = new vscodelc.LanguageClient('Clang Language Server', serverOptions, clientOptions);
+const semanticHighligh

[PATCH] D65637: [clangd] [WIP] Semantic highlighting prototype for the vscode extension.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 213013.
jvikstrom added a comment.

Fixed bug that made it impossible to find colors that were a perfect match to 
the scope we are looking for (ex: fixes function being color as functions and 
not as entity.name)

Explanation for the bug: When adding the function highlighting definition the 
fully qualified name "entity.name.function" was added to a separate list that 
we did not try to match partial scope matches to.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D65637/new/

https://reviews.llvm.org/D65637

Files:
  clang-tools-extra/clangd/clients/clangd-vscode/.gitignore
  clang-tools-extra/clangd/clients/clangd-vscode/src/SemanticHighlighting.ts
  clang-tools-extra/clangd/clients/clangd-vscode/src/TextMate.ts
  clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
  clang-tools-extra/clangd/clients/clangd-vscode/test/extension.test.ts

Index: clang-tools-extra/clangd/clients/clangd-vscode/test/extension.test.ts
===
--- clang-tools-extra/clangd/clients/clangd-vscode/test/extension.test.ts
+++ clang-tools-extra/clangd/clients/clangd-vscode/test/extension.test.ts
@@ -2,14 +2,30 @@
 import * as assert from 'assert';
 
 import * as vscode from 'vscode';
-import * as myExtension from '../src/extension';
+import {SemanticHighlighting} from '../src/SemanticHighlighting';
 
-// TODO: add tests
 suite("Extension Tests", () => {
-
-// Defines a Mocha unit test
-test("Something 1", () => {
-assert.equal(-1, [1, 2, 3].indexOf(5));
-assert.equal(-1, [1, 2, 3].indexOf(0));
+test('HighlightingCache caches incrementaly', () => {
+const feature = new SemanticHighlighting.Feature();
+const uri = 'file://text';
+feature.handleNotification({
+textDocument: {uri: uri},
+lines: [{
+line: 1,
+tokens: 'BAABAAA=',
+}],
+});
+feature.handleNotification({
+textDocument: {uri: uri},
+lines: [{
+line: 2,
+tokens: 'BAABAAA=',
+}],
+});
+//const tokens = feature.tokenCache.files.get(vscode.Uri.parse(uri).toString()).tokens;
+//assert.deepEqual(tokens.get(1), [{character: 4, length: 1, scope: 0}]);
+//assert.deepEqual(tokens.get(2), [{character: 4, length: 1, scope: 0}]);
+
+
 });
 });
\ No newline at end of file
Index: clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
===
--- clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
+++ clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
@@ -1,5 +1,6 @@
 import * as vscode from 'vscode';
 import * as vscodelc from 'vscode-languageclient';
+import { SemanticHighlighting } from './SemanticHighlighting';
 
 /**
  * Method to get workspace configuration option
@@ -12,9 +13,9 @@
 }
 
 namespace SwitchSourceHeaderRequest {
-export const type =
-new vscodelc.RequestType('textDocument/switchSourceHeader');
+export const type =
+new vscodelc.RequestType('textDocument/switchSourceHeader');
 }
 
 class FileStatus {
@@ -32,8 +33,8 @@
 const path = vscode.window.activeTextEditor.document.fileName;
 const status = this.statuses.get(path);
 if (!status) {
-  this.statusBarItem.hide();
-  return;
+this.statusBarItem.hide();
+return;
 }
 this.statusBarItem.text = `clangd: ` + status.state;
 this.statusBarItem.show();
@@ -73,25 +74,30 @@
 // However, VSCode does not have CUDA as a supported language yet, so we
 // cannot add a corresponding activationEvent for CUDA files and clangd will
 // *not* load itself automatically on '.cu' files.
-const cudaFilePattern: string = '**/*.{' +['cu'].join()+ '}';
+const cudaFilePattern: string = '**/*.{' + ['cu'].join() + '}';
 const clientOptions: vscodelc.LanguageClientOptions = {
 // Register the server for c-family and cuda files.
 documentSelector: [
 { scheme: 'file', language: 'c' },
 { scheme: 'file', language: 'cpp' },
-{ scheme: 'file', language: 'objective-c'},
-{ scheme: 'file', language: 'objective-cpp'},
+{ scheme: 'file', language: 'objective-c' },
+{ scheme: 'file', language: 'objective-cpp' },
 { scheme: 'file', pattern: cudaFilePattern },
 ],
 synchronize: !syncFileEvents ? undefined : {
-// FIXME: send sync file events when clangd provides implemenatations.
+// FIXME: send sync file events when clangd provides implemenatations.
 },
 initializationOptions: { clangdFileStatus: true },
 // Do not switch to output window when clangd returns output
   

[PATCH] D65637: [clangd] [WIP] Semantic highlighting prototype for the vscode extension.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom marked an inline comment as done.
jvikstrom added inline comments.



Comment at: clang-tools-extra/clangd/clients/clangd-vscode/src/TextMate.ts:20
+const split = scope.split('.');
+while(split.length > 0) {
+split.splice(-1)

I'm pretty sure this matching logic is wrong. An example of the scopes for 
variables in the Light+ theme:

```
variable.css
variable.scss
variable.other.less
variable.language.wildcard.java
variable.language
storage.type.variable.cs
variable
meta.definition.variable.name
support.variable
entity.name.variable
```

With this kind of matching we have now this means that `variable.other.less` 
will match `variable.other` and `variable.other.less`.
As there is no perfect match for `variable.other.field` it will try to match 
`variable.other` and find the color for less.

What should probably be done is remove the divided map. The query function 
should still be the same but  query on this.colors instead of this.divided.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D65637/new/

https://reviews.llvm.org/D65637



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D65637: [clangd] [WIP] Semantic highlighting prototype for the vscode extension.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 213015.
jvikstrom added a comment.

Fix matching logic on TM scopes.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D65637/new/

https://reviews.llvm.org/D65637

Files:
  clang-tools-extra/clangd/clients/clangd-vscode/.gitignore
  clang-tools-extra/clangd/clients/clangd-vscode/src/SemanticHighlighting.ts
  clang-tools-extra/clangd/clients/clangd-vscode/src/TextMate.ts
  clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
  clang-tools-extra/clangd/clients/clangd-vscode/test/extension.test.ts

Index: clang-tools-extra/clangd/clients/clangd-vscode/test/extension.test.ts
===
--- clang-tools-extra/clangd/clients/clangd-vscode/test/extension.test.ts
+++ clang-tools-extra/clangd/clients/clangd-vscode/test/extension.test.ts
@@ -2,14 +2,30 @@
 import * as assert from 'assert';
 
 import * as vscode from 'vscode';
-import * as myExtension from '../src/extension';
+import {SemanticHighlighting} from '../src/SemanticHighlighting';
 
-// TODO: add tests
 suite("Extension Tests", () => {
-
-// Defines a Mocha unit test
-test("Something 1", () => {
-assert.equal(-1, [1, 2, 3].indexOf(5));
-assert.equal(-1, [1, 2, 3].indexOf(0));
+test('HighlightingCache caches incrementaly', () => {
+const feature = new SemanticHighlighting.Feature();
+const uri = 'file://text';
+feature.handleNotification({
+textDocument: {uri: uri},
+lines: [{
+line: 1,
+tokens: 'BAABAAA=',
+}],
+});
+feature.handleNotification({
+textDocument: {uri: uri},
+lines: [{
+line: 2,
+tokens: 'BAABAAA=',
+}],
+});
+//const tokens = feature.tokenCache.files.get(vscode.Uri.parse(uri).toString()).tokens;
+//assert.deepEqual(tokens.get(1), [{character: 4, length: 1, scope: 0}]);
+//assert.deepEqual(tokens.get(2), [{character: 4, length: 1, scope: 0}]);
+
+
 });
 });
\ No newline at end of file
Index: clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
===
--- clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
+++ clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
@@ -1,5 +1,6 @@
 import * as vscode from 'vscode';
 import * as vscodelc from 'vscode-languageclient';
+import { SemanticHighlighting } from './SemanticHighlighting';
 
 /**
  * Method to get workspace configuration option
@@ -12,9 +13,9 @@
 }
 
 namespace SwitchSourceHeaderRequest {
-export const type =
-new vscodelc.RequestType('textDocument/switchSourceHeader');
+export const type =
+new vscodelc.RequestType('textDocument/switchSourceHeader');
 }
 
 class FileStatus {
@@ -32,8 +33,8 @@
 const path = vscode.window.activeTextEditor.document.fileName;
 const status = this.statuses.get(path);
 if (!status) {
-  this.statusBarItem.hide();
-  return;
+this.statusBarItem.hide();
+return;
 }
 this.statusBarItem.text = `clangd: ` + status.state;
 this.statusBarItem.show();
@@ -73,25 +74,30 @@
 // However, VSCode does not have CUDA as a supported language yet, so we
 // cannot add a corresponding activationEvent for CUDA files and clangd will
 // *not* load itself automatically on '.cu' files.
-const cudaFilePattern: string = '**/*.{' +['cu'].join()+ '}';
+const cudaFilePattern: string = '**/*.{' + ['cu'].join() + '}';
 const clientOptions: vscodelc.LanguageClientOptions = {
 // Register the server for c-family and cuda files.
 documentSelector: [
 { scheme: 'file', language: 'c' },
 { scheme: 'file', language: 'cpp' },
-{ scheme: 'file', language: 'objective-c'},
-{ scheme: 'file', language: 'objective-cpp'},
+{ scheme: 'file', language: 'objective-c' },
+{ scheme: 'file', language: 'objective-cpp' },
 { scheme: 'file', pattern: cudaFilePattern },
 ],
 synchronize: !syncFileEvents ? undefined : {
-// FIXME: send sync file events when clangd provides implemenatations.
+// FIXME: send sync file events when clangd provides implemenatations.
 },
 initializationOptions: { clangdFileStatus: true },
 // Do not switch to output window when clangd returns output
 revealOutputChannelOn: vscodelc.RevealOutputChannelOn.Never
 };
 
-const clangdClient = new vscodelc.LanguageClient('Clang Language Server',serverOptions, clientOptions);
+const clangdClient = new vscodelc.LanguageClient('Clang Language Server', serverOptions, clientOptions);
+const semanticHighlightingFeature = new SemanticHighlighti

[PATCH] D65637: [clangd] [WIP] Semantic highlighting prototype for the vscode extension.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom marked an inline comment as done.
jvikstrom added inline comments.



Comment at: clang-tools-extra/clangd/clients/clangd-vscode/src/TextMate.ts:20
+const split = scope.split('.');
+while(split.length > 0) {
+split.splice(-1)

hokein wrote:
> jvikstrom wrote:
> > I'm pretty sure this matching logic is wrong. An example of the scopes for 
> > variables in the Light+ theme:
> > 
> > ```
> > variable.css
> > variable.scss
> > variable.other.less
> > variable.language.wildcard.java
> > variable.language
> > storage.type.variable.cs
> > variable
> > meta.definition.variable.name
> > support.variable
> > entity.name.variable
> > ```
> > 
> > With this kind of matching we have now this means that 
> > `variable.other.less` will match `variable.other` and `variable.other.less`.
> > As there is no perfect match for `variable.other.field` it will try to 
> > match `variable.other` and find the color for less.
> > 
> > What should probably be done is remove the divided map. The query function 
> > should still be the same but  query on this.colors instead of this.divided.
> The suffix of textmate indicates the language it belongs to (e.g. java), I 
> think we can ignore non-C++ textmates, and merely find the longest-prefix 
> match in all `c++` textmates?
But we have no real way of knowing if a suffix is a language or a more 
"precise" scope.
I don't think it's really possible to differentiate between `variable.other` 
and `variable.css` and say that `variable.css` is not relevant to c++ while 
`variable.other` is. (other than having an exhaustive list of language 
suffixes, which isn't feasible)

The implementation I changed to just saves the full scopes and than when we try 
to find the color of a scope we try to find it in the map, otherwise remove a 
suffix and try to find that scope (over and over)

Ex: Find scope for `variable.other.field.cpp` -> 
  First check for `variable.other.field.cpp` (finds nothing)
  Check `variable.other.field` (finds nothing)
  Check `variable.other` (still finds nothing
  Check `variable` (finds the variable scope and returns that color)

This seems to work, but maybe there's some case I miss. Anyway, there is 
probably a defined order on how you are supposed to find these scopes, I think 
I read a blog post on how VSCode actually does this matching internally. I'll 
try to dig that up and make sure our implementation matches before it's time to 
put up the actual CLs for this.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D65637/new/

https://reviews.llvm.org/D65637



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D65510: [clangd] Fix implicit template instatiations appearing as topLevelDecls.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom marked 2 inline comments as done.
jvikstrom added inline comments.



Comment at: clang-tools-extra/clangd/ClangdUnit.cpp:68
+  if (const auto *TD = dyn_cast(D))
+return TD->getTemplateSpecializationKind() == TSK_ImplicitInstantiation;
+  return false;

ilya-biryukov wrote:
> hokein wrote:
> > ilya-biryukov wrote:
> > > We also want to skip `TSK_ExplicitInstantiationDeclaration` and  
> > > `TSK_ExplicitInstantiationDefinition` here.
> > > This covers cases like (not sure which one of the two enum values we get, 
> > > though):
> > > ```
> > > template 
> > > int foo(T) { ... }
> > > 
> > > template int foo(int); // we'd rather not traverse these, highlightings 
> > > will run into the same problems.
> > > ```
> > > 
> > > Semantics I'm describing are roughly similar to 
> > > `isImplicitInstatiation(D) == !isExplicitInstantion(D)`, where 
> > > `isExplicitInstantiation` is taken from `CodeComplete.cpp`. (If we ignore 
> > > `TSK_Undeclared`, which, I believe, should never be encountered in decls 
> > > passed to HandleTopLevelDecl).
> > > 
> > > Please extract the helper from code complete and this one into a separate 
> > > file (e.g. `clangd/AST.h`) and possibly implement one through the other
> > > Semantics I'm describing are roughly similar to isImplicitInstatiation(D) 
> > > == !isExplicitInstantion(D), 
> > 
> > I think there is a typo here, I believe you mean 
> > `isImplicitInstantiation(D) == !isExplicitSpecialization(D) ` (in 
> > CodeComplete.cpp, it checks whether a Decl is an explicit 
> > **specialization**).
> Yes, there's a typo thanks!
Still want me to move this and the helper from CodeComplete to `AST.h` as it 
isn't used anywhere else? (esp. when we can't implement them through of each 
other)


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D65510/new/

https://reviews.llvm.org/D65510



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D65510: [clangd] Fix implicit template instatiations appearing as topLevelDecls.

[Johan Vikström via Phabricator via cfe-commits]

jvikstrom updated this revision to Diff 213041.
jvikstrom added a comment.

Moved test to ClangdUnitTests.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D65510/new/

https://reviews.llvm.org/D65510

Files:
  clang-tools-extra/clangd/AST.h
  clang-tools-extra/clangd/ClangdUnit.cpp
  clang-tools-extra/clangd/unittests/ClangdUnitTests.cpp


Index: clang-tools-extra/clangd/unittests/ClangdUnitTests.cpp
===
--- clang-tools-extra/clangd/unittests/ClangdUnitTests.cpp
+++ clang-tools-extra/clangd/unittests/ClangdUnitTests.cpp
@@ -103,6 +103,19 @@
   EXPECT_THAT(AST.getLocalTopLevelDecls(), ElementsAre(DeclNamed("main")));
 }
 
+TEST(ClangdUnitTest, DoesNotGetImplicitTemplateTopDecls) {
+  TestTU TU;
+  TU.Code = R"cpp(
+template
+void f(T) {}
+void s() {
+  f(10UL);
+}
+  )cpp";
+  auto AST = TU.build();
+  EXPECT_THAT(AST.getLocalTopLevelDecls(), ElementsAre(DeclNamed("f"), 
DeclNamed("s")));
+}
+
 TEST(ClangdUnitTest, TokensAfterPreamble) {
   TestTU TU;
   TU.AdditionalFiles["foo.h"] = R"(
Index: clang-tools-extra/clangd/ClangdUnit.cpp
===
--- clang-tools-extra/clangd/ClangdUnit.cpp
+++ clang-tools-extra/clangd/ClangdUnit.cpp
@@ -19,8 +19,11 @@
 #include "index/CanonicalIncludes.h"
 #include "index/Index.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclCXX.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceManager.h"
+#include "clang/Basic/Specifiers.h"
 #include "clang/Basic/TokenKinds.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/CompilerInvocation.h"
@@ -60,6 +63,12 @@
   return Vec.capacity() * sizeof(T);
 }
 
+template  bool isImplicitTemplateInstantiation(const Decl *D) {
+  if (const auto *TD = dyn_cast(D))
+return TD->getTemplateSpecializationKind() == TSK_ImplicitInstantiation;
+  return false;
+}
+
 class DeclTrackingASTConsumer : public ASTConsumer {
 public:
   DeclTrackingASTConsumer(std::vector &TopLevelDecls)
@@ -70,6 +79,10 @@
   auto &SM = D->getASTContext().getSourceManager();
   if (!isInsideMainFile(D->getLocation(), SM))
 continue;
+  if (isImplicitTemplateInstantiation(D) ||
+  isImplicitTemplateInstantiation(D) ||
+  isImplicitTemplateInstantiation(D))
+continue;
 
   // ObjCMethodDecl are not actually top-level decls.
   if (isa(D))
Index: clang-tools-extra/clangd/AST.h
===
--- clang-tools-extra/clangd/AST.h
+++ clang-tools-extra/clangd/AST.h
@@ -81,7 +81,7 @@
 /// Example: shortenNamespace("ns1::MyClass", "ns1")
 ///--> "MyClass"
 std::string  shortenNamespace(const llvm::StringRef OriginalName,
-  const llvm::StringRef CurrentNamespace);
+const llvm::StringRef CurrentNamespace);
 
 
 } // namespace clangd


Index: clang-tools-extra/clangd/unittests/ClangdUnitTests.cpp
===
--- clang-tools-extra/clangd/unittests/ClangdUnitTests.cpp
+++ clang-tools-extra/clangd/unittests/ClangdUnitTests.cpp
@@ -103,6 +103,19 @@
   EXPECT_THAT(AST.getLocalTopLevelDecls(), ElementsAre(DeclNamed("main")));
 }
 
+TEST(ClangdUnitTest, DoesNotGetImplicitTemplateTopDecls) {
+  TestTU TU;
+  TU.Code = R"cpp(
+template
+void f(T) {}
+void s() {
+  f(10UL);
+}
+  )cpp";
+  auto AST = TU.build();
+  EXPECT_THAT(AST.getLocalTopLevelDecls(), ElementsAre(DeclNamed("f"), DeclNamed("s")));
+}
+
 TEST(ClangdUnitTest, TokensAfterPreamble) {
   TestTU TU;
   TU.AdditionalFiles["foo.h"] = R"(
Index: clang-tools-extra/clangd/ClangdUnit.cpp
===
--- clang-tools-extra/clangd/ClangdUnit.cpp
+++ clang-tools-extra/clangd/ClangdUnit.cpp
@@ -19,8 +19,11 @@
 #include "index/CanonicalIncludes.h"
 #include "index/Index.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclCXX.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceManager.h"
+#include "clang/Basic/Specifiers.h"
 #include "clang/Basic/TokenKinds.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/CompilerInvocation.h"
@@ -60,6 +63,12 @@
   return Vec.capacity() * sizeof(T);
 }
 
+template  bool isImplicitTemplateInstantiation(const Decl *D) {
+  if (const auto *TD = dyn_cast(D))
+return TD->getTemplateSpecializationKind() == TSK_ImplicitInstantiation;
+  return false;
+}
+
 class DeclTrackingASTConsumer : public ASTConsumer {
 public:
   DeclTrackingASTConsumer(std::vector &TopLevelDecls)
@@ -70,6 +79,10 @@
   auto &SM = D->getASTContext().getSourceManager();
   if (!isInsideMainFile(D->getLocation(), SM))
 continue;
+  if (isImplicitTemplateInstantiation(D) ||
+