jvikstrom updated this revision to Diff 210567.
jvikstrom edited the summary of this revision.
jvikstrom added a comment.
Added additional note to FIXME on how to solve and also move(ing) Old.
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D64475/new/
https://reviews.llvm.org/D64475
Files:
clang-tools-extra/clangd/ClangdLSPServer.cpp
clang-tools-extra/clangd/ClangdLSPServer.h
clang-tools-extra/clangd/SemanticHighlighting.cpp
clang-tools-extra/clangd/SemanticHighlighting.h
clang-tools-extra/clangd/test/semantic-highlighting.test
clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===================================================================
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,9 +29,12 @@
return Tokens;
}
-void checkHighlightings(llvm::StringRef Code) {
- Annotations Test(Code);
+std::vector<HighlightingToken> getActualTokens(Annotations &Test) {
auto AST = TestTU::withCode(Test.code()).build();
+ return getSemanticHighlightings(AST);
+}
+
+std::vector<HighlightingToken> getExpectedTokens(Annotations &Test) {
static const std::map<HighlightingKind, std::string> KindToString{
{HighlightingKind::Variable, "Variable"},
{HighlightingKind::Function, "Function"},
@@ -48,9 +51,43 @@
Test.ranges(KindString.second), KindString.first);
ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
}
+ llvm::sort(ExpectedTokens);
+ return ExpectedTokens;
+}
- auto ActualTokens = getSemanticHighlightings(AST);
- EXPECT_THAT(ActualTokens, testing::UnorderedElementsAreArray(ExpectedTokens));
+std::vector<int> getExpectedEmptyLines(Annotations &Test) {
+ auto EmptyRanges = Test.ranges("Empty");
+ std::vector<int> EmptyLines;
+ EmptyLines.reserve(EmptyRanges.size());
+ for (const auto &EmptyRange : EmptyRanges)
+ EmptyLines.push_back(EmptyRange.start.line);
+ return EmptyLines;
+}
+
+void checkHighlightings(llvm::StringRef Code) {
+ Annotations Test(Code);
+ EXPECT_THAT(getActualTokens(Test),
+ getExpectedTokens(Test));
+}
+
+void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) {
+ Annotations OldTest(OldCode);
+ Annotations NewTest(NewCode);
+ std::vector<LineHighlightings> ExpectedLinePairHighlighting;
+ for (int Line : getExpectedEmptyLines(NewTest))
+ ExpectedLinePairHighlighting.push_back({Line, {}});
+ std::map<int, std::vector<HighlightingToken>> ExpectedLines;
+ for (const HighlightingToken &Token : getExpectedTokens(NewTest))
+ ExpectedLines[Token.R.start.line].push_back(Token);
+ for (auto &LineTokens : ExpectedLines) {
+ ExpectedLinePairHighlighting.push_back(
+ {LineTokens.first, LineTokens.second});
+ }
+
+ std::vector<LineHighlightings> ActualDiffed =
+ diffHighlightings(getActualTokens(NewTest), getActualTokens(OldTest));
+ EXPECT_THAT(ActualDiffed,
+ testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting));
}
TEST(SemanticHighlighting, GetsCorrectTokens) {
@@ -251,21 +288,110 @@
return Pos;
};
- std::vector<HighlightingToken> Tokens{
- {HighlightingKind::Variable,
- Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
- {HighlightingKind::Function,
- Range{CreatePosition(3, 4), CreatePosition(3, 7)}},
- {HighlightingKind::Variable,
- Range{CreatePosition(1, 1), CreatePosition(1, 5)}}};
+ std::vector<LineHighlightings> Tokens{
+ {3,
+ {{HighlightingKind::Variable,
+ Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
+ {HighlightingKind::Function,
+ Range{CreatePosition(3, 4), CreatePosition(3, 7)}}}},
+ {1,
+ {{HighlightingKind::Variable,
+ Range{CreatePosition(1, 1), CreatePosition(1, 5)}}}}};
std::vector<SemanticHighlightingInformation> ActualResults =
toSemanticHighlightingInformation(Tokens);
std::vector<SemanticHighlightingInformation> ExpectedResults = {
- {1, "AAAAAQAEAAA="},
- {3, "AAAACAAEAAAAAAAEAAMAAQ=="}};
+ {3, "AAAACAAEAAAAAAAEAAMAAQ=="}, {1, "AAAAAQAEAAA="}};
EXPECT_EQ(ActualResults, ExpectedResults);
}
+TEST(SemanticHighlighting, HighlightingDiffer) {
+ struct {
+ llvm::StringRef OldCode;
+ llvm::StringRef NewCode;
+ } TestCases[]{{
+ R"cpp(
+ int A
+ double B;
+ struct C {};
+ )cpp",
+ R"cpp(
+ int A;
+ double B;
+ struct C {};
+ )cpp"},
+ {
+ R"cpp(
+ struct Alpha {
+ double SomeVariable = 9483.301;
+ };
+ struct Beta {};
+ int A = 121;
+ Alpha Var;
+ )cpp",
+ R"cpp(
+ struct Alpha {
+ double SomeVariable = 9483.301;
+ };
+ struct Beta {}; // Some comment
+ $Variable[[intA]] = 121;
+ $Class[[Beta]] $Variable[[Var]];
+ )cpp"},
+ {
+ R"cpp(
+ int A = 121; int B;
+ )cpp",
+ R"cpp(
+ $Variable[[intA]] = 121; int $Variable[[B]];
+ )cpp"},
+ {
+ R"cpp(
+ int A;
+ )cpp",
+ R"cpp(
+ struct $Class[[A]];
+ )cpp"},
+ {
+ R"cpp(
+ int ABBA = 23;
+ double C = 123.3 * 5;
+ )cpp",
+ R"cpp(
+ int ABBA = 23;
+ $Empty[[]]//double C = 123.3 * 5;
+ float $Variable[[H]];
+ )cpp"},
+ {
+ R"cpp(
+ int a;
+ int b;
+ int c;
+ )cpp",
+ R"cpp(
+ int a;
+ int $Variable[[new_b]];
+ int c;
+ )cpp"
+ },
+ {
+ R"cpp(
+ int a;
+ int b;
+ int c;
+ int d;
+ )cpp",
+ R"cpp(
+ int a;
+/* $Empty[[]] int b;
+ $Empty[[]]int c;*/
+ int d;
+ )cpp"
+ }};
+
+ for (const auto &Test : TestCases) {
+ checkDiffedHighlights(Test.OldCode, Test.NewCode);
+ }
+}
+
} // namespace
} // namespace clangd
} // namespace clang
Index: clang-tools-extra/clangd/test/semantic-highlighting.test
===================================================================
--- clang-tools-extra/clangd/test/semantic-highlighting.test
+++ clang-tools-extra/clangd/test/semantic-highlighting.test
@@ -49,6 +49,40 @@
# CHECK-NEXT: }
# CHECK-NEXT:}
---
+{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo2.cpp","languageId":"cpp","version":1,"text":"int x = 2;\nint y = 2;"}}}
+# CHECK: "method": "textDocument/semanticHighlighting",
+# CHECK-NEXT: "params": {
+# CHECK-NEXT: "lines": [
+# CHECK-NEXT: {
+# CHECK-NEXT: "line": 0,
+# CHECK-NEXT: "tokens": "AAAABAABAAA="
+# CHECK-NEXT: }
+# CHECK-NEXT: {
+# CHECK-NEXT: "line": 1,
+# CHECK-NEXT: "tokens": "AAAABAABAAA="
+# CHECK-NEXT: }
+# CHECK-NEXT: ],
+# CHECK-NEXT: "textDocument": {
+# CHECK-NEXT: "uri": "file:///clangd-test/foo2.cpp"
+# CHECK-NEXT: }
+# CHECK-NEXT: }
+# CHECK-NEXT:}
+---
+{"jsonrpc":"2.0","method":"textDocument/didChange","params":{"textDocument":{"uri":"test:///foo.cpp","version":2},"contentChanges": [{"range":{"start": {"line": 0,"character": 6},"end": {"line": 0,"character": 6}},"rangeLength": 0,"text": "\nint y = 2;"}]}}
+# CHECK: "method": "textDocument/semanticHighlighting",
+# CHECK-NEXT: "params": {
+# CHECK-NEXT: "lines": [
+# CHECK-NEXT: {
+# CHECK-NEXT: "line": 1,
+# CHECK-NEXT: "tokens": "AAAABAABAAA="
+# CHECK-NEXT: }
+# CHECK-NEXT: ],
+# CHECK-NEXT: "textDocument": {
+# CHECK-NEXT: "uri": "file:///clangd-test/foo.cpp"
+# CHECK-NEXT: }
+# CHECK-NEXT: }
+# CHECK-NEXT:}
+---
{"jsonrpc":"2.0","id":3,"method":"shutdown"}
---
{"jsonrpc":"2.0","method":"exit"}
Index: clang-tools-extra/clangd/SemanticHighlighting.h
===================================================================
--- clang-tools-extra/clangd/SemanticHighlighting.h
+++ clang-tools-extra/clangd/SemanticHighlighting.h
@@ -43,7 +43,16 @@
Range R;
};
-bool operator==(const HighlightingToken &Lhs, const HighlightingToken &Rhs);
+bool operator==(const HighlightingToken &L, const HighlightingToken &R);
+bool operator<(const HighlightingToken &L, const HighlightingToken &R);
+
+/// Contains all information about highlightings on a single line.
+struct LineHighlightings {
+ int Line;
+ std::vector<HighlightingToken> Tokens;
+};
+
+bool operator==(const LineHighlightings &L, const LineHighlightings &R);
// Returns all HighlightingTokens from an AST. Only generates highlights for the
// main AST.
@@ -53,9 +62,22 @@
/// (https://manual.macromates.com/en/language_grammars).
llvm::StringRef toTextMateScope(HighlightingKind Kind);
-// Convert to LSP's semantic highlighting information.
+/// Convert to LSP's semantic highlighting information.
std::vector<SemanticHighlightingInformation>
-toSemanticHighlightingInformation(llvm::ArrayRef<HighlightingToken> Tokens);
+toSemanticHighlightingInformation(llvm::ArrayRef<LineHighlightings> Tokens);
+
+// Return a line-by-line diff between two highlightings.
+// - if the tokens on a line are the same in both hightlightings this line is
+// omitted.
+// - if a line exists in NewHighligtings but not in OldHighligtings the tokens
+// on this line is emitted., we emit the tokens on this line
+// - if a line not exists in NewHighligtings but in OldHighligtings an empty
+// line is emitted (to tell client to clear the previous highlightings on this
+// line)
+/// REQUIRED: OldHighlightings and NewHighlightings are sorted.
+std::vector<LineHighlightings>
+diffHighlightings(ArrayRef<HighlightingToken> NewHighlightings,
+ ArrayRef<HighlightingToken> OldHighlightings);
} // namespace clangd
} // namespace clang
Index: clang-tools-extra/clangd/SemanticHighlighting.cpp
===================================================================
--- clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -33,10 +33,7 @@
TraverseAST(Ctx);
// Initializer lists can give duplicates of tokens, therefore all tokens
// must be deduplicated.
- llvm::sort(Tokens,
- [](const HighlightingToken &L, const HighlightingToken &R) {
- return std::tie(L.R, L.Kind) < std::tie(R.R, R.Kind);
- });
+ llvm::sort(Tokens);
auto Last = std::unique(Tokens.begin(), Tokens.end());
Tokens.erase(Last, Tokens.end());
return Tokens;
@@ -260,10 +257,60 @@
llvm::support::endian::write16be(Buf.data(), I);
OS.write(Buf.data(), Buf.size());
}
+
+// Get the highlightings on \c Line where the first entry of line is at \c
+// StartLineIt. If it is not at \c StartLineIt an empty vector is returned.
+ArrayRef<HighlightingToken>
+takeLine(ArrayRef<HighlightingToken> AllTokens,
+ ArrayRef<HighlightingToken>::iterator StartLineIt, int Line) {
+ return ArrayRef<HighlightingToken>(StartLineIt, AllTokens.end())
+ .take_while([Line](const HighlightingToken &Token) {
+ return Token.R.start.line == Line;
+ });
+}
} // namespace
-bool operator==(const HighlightingToken &Lhs, const HighlightingToken &Rhs) {
- return Lhs.Kind == Rhs.Kind && Lhs.R == Rhs.R;
+std::vector<LineHighlightings>
+diffHighlightings(ArrayRef<HighlightingToken> NewHighlightings,
+ ArrayRef<HighlightingToken> OldHighlightings) {
+ // FIXME: There's an edge case when tokens span multiple lines. If the first
+ // token on the line started on a line above the current one and the rest of
+ // the line is the equal to the previous one than we will remove all
+ // highlights but the ones for the token spanning multiple lines. This means
+ // that when we get into the LSP layer the only highlights that will be
+ // visible are the ones for the token spanning multiple lines.
+ // Example:
+ // EndOfMultilineToken Token Token Token
+ // If "Token Token Token" don't differ from previously the line is
+ // incorrectly removed. Suggestion to fix is to separate any multiline tokens
+ // into one token for every line it covers. This requires reading from the
+ // file buffer to figure out the length of each line though.
+ std::vector<LineHighlightings> LineTokens;
+ // ArrayRefs to the current line in the highlights.
+ ArrayRef<HighlightingToken> NewLine(NewHighlightings.begin(),
+ /*length*/0UL);
+ ArrayRef<HighlightingToken> OldLine(OldHighlightings.begin(),
+ /*length*/0UL);
+ auto NewEnd = NewHighlightings.end();
+ auto OldEnd = OldHighlightings.end();
+ for (int Line = 0; NewLine.end() < NewEnd || OldLine.end() < OldEnd; ++Line) {
+ NewLine = takeLine(NewHighlightings, NewLine.end(), Line);
+ OldLine = takeLine(OldHighlightings, OldLine.end(), Line);
+ if (NewLine != OldLine)
+ LineTokens.push_back({Line, NewLine});
+ }
+
+ return LineTokens;
+}
+
+bool operator==(const HighlightingToken &L, const HighlightingToken &R) {
+ return std::tie(L.R, L.Kind) == std::tie(R.R, R.Kind);
+}
+bool operator<(const HighlightingToken &L, const HighlightingToken &R) {
+ return std::tie(L.R, L.Kind) < std::tie(R.R, R.Kind);
+}
+bool operator==(const LineHighlightings &L, const LineHighlightings &R) {
+ return std::tie(L.Line, L.Tokens) == std::tie(R.Line, R.Tokens);
}
std::vector<HighlightingToken> getSemanticHighlightings(ParsedAST &AST) {
@@ -271,22 +318,18 @@
}
std::vector<SemanticHighlightingInformation>
-toSemanticHighlightingInformation(llvm::ArrayRef<HighlightingToken> Tokens) {
+toSemanticHighlightingInformation(llvm::ArrayRef<LineHighlightings> Tokens) {
if (Tokens.size() == 0)
return {};
// FIXME: Tokens might be multiple lines long (block comments) in this case
// this needs to add multiple lines for those tokens.
- std::map<int, std::vector<HighlightingToken>> TokenLines;
- for (const HighlightingToken &Token : Tokens)
- TokenLines[Token.R.start.line].push_back(Token);
-
std::vector<SemanticHighlightingInformation> Lines;
- Lines.reserve(TokenLines.size());
- for (const auto &Line : TokenLines) {
+ Lines.reserve(Tokens.size());
+ for (const auto &Line : Tokens) {
llvm::SmallVector<char, 128> LineByteTokens;
llvm::raw_svector_ostream OS(LineByteTokens);
- for (const auto &Token : Line.second) {
+ for (const auto &Token : Line.Tokens) {
// Writes the token to LineByteTokens in the byte format specified by the
// LSP proposal. Described below.
// |<---- 4 bytes ---->|<-- 2 bytes -->|<--- 2 bytes -->|
@@ -297,7 +340,7 @@
write16be(static_cast<int>(Token.Kind), OS);
}
- Lines.push_back({Line.first, encodeBase64(LineByteTokens)});
+ Lines.push_back({Line.Line, encodeBase64(LineByteTokens)});
}
return Lines;
Index: clang-tools-extra/clangd/ClangdLSPServer.h
===================================================================
--- clang-tools-extra/clangd/ClangdLSPServer.h
+++ clang-tools-extra/clangd/ClangdLSPServer.h
@@ -136,6 +136,8 @@
DiagnosticToReplacementMap;
/// Caches FixIts per file and diagnostics
llvm::StringMap<DiagnosticToReplacementMap> FixItsMap;
+ std::mutex HighlightingsMutex;
+ llvm::StringMap<std::vector<HighlightingToken>> FileToHighlightings;
// Most code should not deal with Transport directly.
// MessageHandler deals with incoming messages, use call() etc for outgoing.
Index: clang-tools-extra/clangd/ClangdLSPServer.cpp
===================================================================
--- clang-tools-extra/clangd/ClangdLSPServer.cpp
+++ clang-tools-extra/clangd/ClangdLSPServer.cpp
@@ -452,6 +452,12 @@
void ClangdLSPServer::onDocumentDidOpen(
const DidOpenTextDocumentParams &Params) {
PathRef File = Params.textDocument.uri.file();
+ {
+ // If a file was closed before highlightings were generated for the last
+ // edit there are stale previous highlightings.
+ std::lock_guard<std::mutex> Lock(HighlightingsMutex);
+ FileToHighlightings.erase(File);
+ }
const std::string &Contents = Params.textDocument.text;
@@ -601,6 +607,10 @@
std::lock_guard<std::mutex> Lock(FixItsMutex);
FixItsMap.erase(File);
}
+ {
+ std::lock_guard<std::mutex> HLock(HighlightingsMutex);
+ FileToHighlightings.erase(File);
+ }
// clangd will not send updates for this file anymore, so we empty out the
// list of diagnostics shown on the client (e.g. in the "Problems" pane of
// VSCode). Note that this cannot race with actual diagnostics responses
@@ -1099,9 +1109,21 @@
void ClangdLSPServer::onHighlightingsReady(
PathRef File, std::vector<HighlightingToken> Highlightings) {
+ std::vector<HighlightingToken> Old;
+ {
+ std::lock_guard<std::mutex> Lock(HighlightingsMutex);
+ Old = std::move(FileToHighlightings[File]);
+ }
+ // LSP allows us to send incremental edits of highlightings. Also need to diff
+ // to remove highlightings from tokens that should no longer have them.
+ std::vector<LineHighlightings> Diffed = diffHighlightings(Highlightings, Old);
+ {
+ std::lock_guard<std::mutex> Lock(HighlightingsMutex);
+ FileToHighlightings[File] = std::move(Highlightings);
+ }
publishSemanticHighlighting(
{{URIForFile::canonicalize(File, /*TUPath=*/File)},
- toSemanticHighlightingInformation(Highlightings)});
+ toSemanticHighlightingInformation(Diffed)});
}
void ClangdLSPServer::onDiagnosticsReady(PathRef File,
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
