https://github.com/tcottin created https://github.com/llvm/llvm-project/pull/150790
Followup work of #140498 to continue the work on clangd/clangd#529 Introduce the use of the Clang doxygen parser to parse the documentation of hovered code. - ASTContext independent doxygen parsing - Parsing doxygen commands to markdown for hover information Note: after this PR I have planned another patch to rearrange the information shown in the hover info. This PR is just for the basic introduction of doxygen parsing for hover information. >From 78c7cc2441b9395fd2a0b3ac6b25090dd7937098 Mon Sep 17 00:00:00 2001 From: Tim Cottin <timcot...@gmx.de> Date: Sat, 26 Jul 2025 18:25:45 +0000 Subject: [PATCH] [clangd] introduce doxygen parser --- clang-tools-extra/clangd/CMakeLists.txt | 1 + .../clangd/CodeCompletionStrings.cpp | 15 +- clang-tools-extra/clangd/Hover.cpp | 187 +++++++++++++- clang-tools-extra/clangd/Hover.h | 13 +- .../clangd/SymbolDocumentation.cpp | 221 +++++++++++++++++ .../clangd/SymbolDocumentation.h | 140 +++++++++++ clang-tools-extra/clangd/support/Markup.cpp | 7 +- .../clangd/unittests/CMakeLists.txt | 1 + .../clangd/unittests/HoverTests.cpp | 229 ++++++++++++++++++ .../unittests/SymbolDocumentationTests.cpp | 161 ++++++++++++ .../clangd/unittests/support/MarkupTests.cpp | 2 + clang/include/clang/AST/Comment.h | 21 ++ clang/include/clang/AST/CommentSema.h | 1 + clang/lib/AST/CommentParser.cpp | 5 +- clang/lib/AST/CommentSema.cpp | 7 +- .../clang-tools-extra/clangd/BUILD.gn | 1 + 16 files changed, 993 insertions(+), 19 deletions(-) create mode 100644 clang-tools-extra/clangd/SymbolDocumentation.cpp create mode 100644 clang-tools-extra/clangd/SymbolDocumentation.h create mode 100644 clang-tools-extra/clangd/unittests/SymbolDocumentationTests.cpp diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt index a1e9da41b4b32..06920a97ddc88 100644 --- a/clang-tools-extra/clangd/CMakeLists.txt +++ b/clang-tools-extra/clangd/CMakeLists.txt @@ -108,6 +108,7 @@ add_clang_library(clangDaemon STATIC SemanticHighlighting.cpp SemanticSelection.cpp SourceCode.cpp + SymbolDocumentation.cpp SystemIncludeExtractor.cpp TidyProvider.cpp TUScheduler.cpp diff --git a/clang-tools-extra/clangd/CodeCompletionStrings.cpp b/clang-tools-extra/clangd/CodeCompletionStrings.cpp index 9b4442b0bb76f..196a1624e1c04 100644 --- a/clang-tools-extra/clangd/CodeCompletionStrings.cpp +++ b/clang-tools-extra/clangd/CodeCompletionStrings.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "CodeCompletionStrings.h" +#include "Config.h" #include "clang-c/Index.h" #include "clang/AST/ASTContext.h" #include "clang/AST/RawCommentList.h" @@ -100,7 +101,19 @@ std::string getDeclComment(const ASTContext &Ctx, const NamedDecl &Decl) { // the comments for namespaces. return ""; } - const RawComment *RC = getCompletionComment(Ctx, &Decl); + + const RawComment *RC = nullptr; + const Config &Cfg = Config::current(); + + if (Cfg.Documentation.CommentFormat == Config::CommentFormatPolicy::Doxygen && + isa<ParmVarDecl>(Decl)) { + // Parameters are documented in the function comment. + if (const auto *FD = dyn_cast<FunctionDecl>(Decl.getDeclContext())) + RC = getCompletionComment(Ctx, FD); + } else { + RC = getCompletionComment(Ctx, &Decl); + } + if (!RC) return ""; // Sanity check that the comment does not come from the PCH. We choose to not diff --git a/clang-tools-extra/clangd/Hover.cpp b/clang-tools-extra/clangd/Hover.cpp index 1e0718d673260..63fdc7c24a7a8 100644 --- a/clang-tools-extra/clangd/Hover.cpp +++ b/clang-tools-extra/clangd/Hover.cpp @@ -18,6 +18,7 @@ #include "Protocol.h" #include "Selection.h" #include "SourceCode.h" +#include "SymbolDocumentation.h" #include "clang-include-cleaner/Analysis.h" #include "clang-include-cleaner/IncludeSpeller.h" #include "clang-include-cleaner/Types.h" @@ -41,6 +42,7 @@ #include "clang/AST/Type.h" #include "clang/Basic/CharInfo.h" #include "clang/Basic/LLVM.h" +#include "clang/Basic/LangOptions.h" #include "clang/Basic/SourceLocation.h" #include "clang/Basic/SourceManager.h" #include "clang/Basic/Specifiers.h" @@ -627,6 +629,9 @@ HoverInfo getHoverContents(const NamedDecl *D, const PrintingPolicy &PP, HI.Name = printName(Ctx, *D); const auto *CommentD = getDeclForComment(D); HI.Documentation = getDeclComment(Ctx, *CommentD); + // safe the language options to be able to create the comment::CommandTraits + // to parse the documentation + HI.CommentOpts = D->getASTContext().getLangOpts().CommentOpts; enhanceFromIndex(HI, *CommentD, Index); if (HI.Documentation.empty()) HI.Documentation = synthesizeDocumentation(D); @@ -1388,9 +1393,170 @@ static std::string formatOffset(uint64_t OffsetInBits) { return Offset; } -markup::Document HoverInfo::present() const { +markup::Document HoverInfo::presentDoxygen() const { + // NOTE: this function is currently almost identical to presentDefault(). + // This is to have a minimal change when introducing the doxygen parser. + // This function will be changed when rearranging the output for doxygen + // parsed documentation. + markup::Document Output; + // Header contains a text of the form: + // variable `var` + // + // class `X` + // + // function `foo` + // + // expression + // + // Note that we are making use of a level-3 heading because VSCode renders + // level 1 and 2 headers in a huge font, see + // https://github.com/microsoft/vscode/issues/88417 for details. + markup::Paragraph &Header = Output.addHeading(3); + if (Kind != index::SymbolKind::Unknown) + Header.appendText(index::getSymbolKindString(Kind)).appendSpace(); + assert(!Name.empty() && "hover triggered on a nameless symbol"); + + Header.appendCode(Name); + + if (!Provider.empty()) { + markup::Paragraph &DI = Output.addParagraph(); + DI.appendText("provided by"); + DI.appendSpace(); + DI.appendCode(Provider); + Output.addRuler(); + } + + // Put a linebreak after header to increase readability. + Output.addRuler(); + // Print Types on their own lines to reduce chances of getting line-wrapped by + // editor, as they might be long. + if (ReturnType) { + // For functions we display signature in a list form, e.g.: + // → `x` + // Parameters: + // - `bool param1` + // - `int param2 = 5` + Output.addParagraph().appendText("→ ").appendCode( + llvm::to_string(*ReturnType)); + } + + SymbolDocCommentVisitor SymbolDoc(Documentation, CommentOpts); + + if (Parameters && !Parameters->empty()) { + Output.addParagraph().appendText("Parameters:"); + markup::BulletList &L = Output.addBulletList(); + for (const auto &Param : *Parameters) { + markup::Paragraph &P = L.addItem().addParagraph(); + P.appendCode(llvm::to_string(Param)); + + if (SymbolDoc.isParameterDocumented(llvm::to_string(Param.Name))) { + P.appendText(" -"); + SymbolDoc.parameterDocToMarkup(llvm::to_string(Param.Name), P); + } + } + } + // Don't print Type after Parameters or ReturnType as this will just duplicate + // the information + if (Type && !ReturnType && !Parameters) + Output.addParagraph().appendText("Type: ").appendCode( + llvm::to_string(*Type)); + + if (Value) { + markup::Paragraph &P = Output.addParagraph(); + P.appendText("Value = "); + P.appendCode(*Value); + } + + if (Offset) + Output.addParagraph().appendText("Offset: " + formatOffset(*Offset)); + if (Size) { + auto &P = Output.addParagraph().appendText("Size: " + formatSize(*Size)); + if (Padding && *Padding != 0) { + P.appendText( + llvm::formatv(" (+{0} padding)", formatSize(*Padding)).str()); + } + if (Align) + P.appendText(", alignment " + formatSize(*Align)); + } + + if (CalleeArgInfo) { + assert(CallPassType); + std::string Buffer; + llvm::raw_string_ostream OS(Buffer); + OS << "Passed "; + if (CallPassType->PassBy != HoverInfo::PassType::Value) { + OS << "by "; + if (CallPassType->PassBy == HoverInfo::PassType::ConstRef) + OS << "const "; + OS << "reference "; + } + if (CalleeArgInfo->Name) + OS << "as " << CalleeArgInfo->Name; + else if (CallPassType->PassBy == HoverInfo::PassType::Value) + OS << "by value"; + if (CallPassType->Converted && CalleeArgInfo->Type) + OS << " (converted to " << CalleeArgInfo->Type->Type << ")"; + Output.addParagraph().appendText(OS.str()); + } + if (Kind == index::SymbolKind::Parameter) { + if (SymbolDoc.isParameterDocumented(Name)) + SymbolDoc.parameterDocToMarkup(Name, Output.addParagraph()); + } else + SymbolDoc.docToMarkup(Output); + + if (!Definition.empty()) { + Output.addRuler(); + std::string Buffer; + + if (!Definition.empty()) { + // Append scope comment, dropping trailing "::". + // Note that we don't print anything for global namespace, to not annoy + // non-c++ projects or projects that are not making use of namespaces. + if (!LocalScope.empty()) { + // Container name, e.g. class, method, function. + // We might want to propagate some info about container type to print + // function foo, class X, method X::bar, etc. + Buffer += + "// In " + llvm::StringRef(LocalScope).rtrim(':').str() + '\n'; + } else if (NamespaceScope && !NamespaceScope->empty()) { + Buffer += "// In namespace " + + llvm::StringRef(*NamespaceScope).rtrim(':').str() + '\n'; + } + + if (!AccessSpecifier.empty()) { + Buffer += AccessSpecifier + ": "; + } + + Buffer += Definition; + } + + Output.addCodeBlock(Buffer, DefinitionLanguage); + } + + if (!UsedSymbolNames.empty()) { + Output.addRuler(); + markup::Paragraph &P = Output.addParagraph(); + P.appendText("provides "); + + const std::vector<std::string>::size_type SymbolNamesLimit = 5; + auto Front = llvm::ArrayRef(UsedSymbolNames).take_front(SymbolNamesLimit); + + llvm::interleave( + Front, [&](llvm::StringRef Sym) { P.appendCode(Sym); }, + [&] { P.appendText(", "); }); + if (UsedSymbolNames.size() > Front.size()) { + P.appendText(" and "); + P.appendText(std::to_string(UsedSymbolNames.size() - Front.size())); + P.appendText(" more"); + } + } + return Output; +} + +markup::Document HoverInfo::presentDefault() const { + markup::Document Output; // Header contains a text of the form: // variable `var` // @@ -1538,21 +1704,22 @@ markup::Document HoverInfo::present() const { std::string HoverInfo::present(MarkupKind Kind) const { if (Kind == MarkupKind::Markdown) { const Config &Cfg = Config::current(); - if ((Cfg.Documentation.CommentFormat == - Config::CommentFormatPolicy::Markdown) || - (Cfg.Documentation.CommentFormat == - Config::CommentFormatPolicy::Doxygen)) - // If the user prefers Markdown, we use the present() method to generate - // the Markdown output. - return present().asMarkdown(); + if (Cfg.Documentation.CommentFormat == + Config::CommentFormatPolicy::Markdown) + return presentDefault().asMarkdown(); + if (Cfg.Documentation.CommentFormat == + Config::CommentFormatPolicy::Doxygen) { + std::string T = presentDoxygen().asMarkdown(); + return T; + } if (Cfg.Documentation.CommentFormat == Config::CommentFormatPolicy::PlainText) // If the user prefers plain text, we use the present() method to generate // the plain text output. - return present().asEscapedMarkdown(); + return presentDefault().asEscapedMarkdown(); } - return present().asPlainText(); + return presentDefault().asPlainText(); } // If the backtick at `Offset` starts a probable quoted range, return the range diff --git a/clang-tools-extra/clangd/Hover.h b/clang-tools-extra/clangd/Hover.h index 2f65431bd72de..2578e7a4339d0 100644 --- a/clang-tools-extra/clangd/Hover.h +++ b/clang-tools-extra/clangd/Hover.h @@ -74,6 +74,8 @@ struct HoverInfo { std::optional<Range> SymRange; index::SymbolKind Kind = index::SymbolKind::Unknown; std::string Documentation; + // required to create a comments::CommandTraits object without the ASTContext + CommentOptions CommentOpts; /// Source code containing the definition of the symbol. std::string Definition; const char *DefinitionLanguage = "cpp"; @@ -118,10 +120,15 @@ struct HoverInfo { // alphabetical order. std::vector<std::string> UsedSymbolNames; - /// Produce a user-readable information. - markup::Document present() const; - + /// Produce a user-readable information based on the specified markup kind. std::string present(MarkupKind Kind) const; + +private: + /// Parse and render the hover information as Doxygen documentation. + markup::Document presentDoxygen() const; + + /// Render the hover information as a default documentation. + markup::Document presentDefault() const; }; inline bool operator==(const HoverInfo::PrintedType &LHS, diff --git a/clang-tools-extra/clangd/SymbolDocumentation.cpp b/clang-tools-extra/clangd/SymbolDocumentation.cpp new file mode 100644 index 0000000000000..1c14ccb01fc26 --- /dev/null +++ b/clang-tools-extra/clangd/SymbolDocumentation.cpp @@ -0,0 +1,221 @@ +//===--- SymbolDocumentation.cpp ==-------------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "SymbolDocumentation.h" + +#include "support/Markup.h" +#include "clang/AST/Comment.h" +#include "clang/AST/CommentCommandTraits.h" +#include "clang/AST/CommentVisitor.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringRef.h" + +namespace clang { +namespace clangd { + +void commandToMarkup(markup::Paragraph &Out, StringRef Command, + comments::CommandMarkerKind CommandMarker, + StringRef Args) { + Out.appendBoldText( + (CommandMarker == (comments::CommandMarkerKind::CMK_At) ? "@" : "\\") + + Command.str()); + if (!Args.empty()) { + Out.appendSpace(); + Out.appendEmphasizedText(Args.str()); + } +} + +class ParagraphToMarkupDocument + : public comments::ConstCommentVisitor<ParagraphToMarkupDocument> { +public: + ParagraphToMarkupDocument(markup::Paragraph &Out, + const comments::CommandTraits &Traits) + : Out(Out), Traits(Traits) {} + + void visitParagraphComment(const comments::ParagraphComment *C) { + if (!C) + return; + + for (const auto *Child = C->child_begin(); Child != C->child_end(); + ++Child) { + visit(*Child); + } + } + + void visitTextComment(const comments::TextComment *C) { + // Always trim leading space after a newline. + StringRef Text = C->getText(); + if (LastChunkEndsWithNewline && C->getText().starts_with(' ')) + Text = Text.drop_front(); + + LastChunkEndsWithNewline = C->hasTrailingNewline(); + Out.appendText(Text.str() + (LastChunkEndsWithNewline ? "\n" : "")); + } + + void visitInlineCommandComment(const comments::InlineCommandComment *C) { + + if (C->getNumArgs() > 0) { + std::string ArgText; + for (unsigned I = 0; I < C->getNumArgs(); ++I) { + if (!ArgText.empty()) + ArgText += " "; + ArgText += C->getArgText(I); + } + + switch (C->getRenderKind()) { + case comments::InlineCommandRenderKind::Monospaced: + Out.appendCode(ArgText); + break; + case comments::InlineCommandRenderKind::Bold: + Out.appendBoldText(ArgText); + break; + case comments::InlineCommandRenderKind::Emphasized: + Out.appendEmphasizedText(ArgText); + break; + default: + commandToMarkup(Out, C->getCommandName(Traits), C->getCommandMarker(), + ArgText); + break; + } + } else { + if (C->getCommandName(Traits) == "n") { + // \n is a special case, it is used to create a new line. + Out.appendText(" \n"); + LastChunkEndsWithNewline = true; + return; + } + + commandToMarkup(Out, C->getCommandName(Traits), C->getCommandMarker(), + ""); + } + } + + void visitHTMLStartTagComment(const comments::HTMLStartTagComment *STC) { + std::string TagText = "<" + STC->getTagName().str(); + + for (unsigned I = 0; I < STC->getNumAttrs(); ++I) { + const comments::HTMLStartTagComment::Attribute &Attr = STC->getAttr(I); + TagText += " " + Attr.Name.str() + "=\"" + Attr.Value.str() + "\""; + } + + if (STC->isSelfClosing()) + TagText += " /"; + TagText += ">"; + + LastChunkEndsWithNewline = STC->hasTrailingNewline(); + Out.appendText(TagText + (LastChunkEndsWithNewline ? "\n" : "")); + } + + void visitHTMLEndTagComment(const comments::HTMLEndTagComment *ETC) { + LastChunkEndsWithNewline = ETC->hasTrailingNewline(); + Out.appendText("</" + ETC->getTagName().str() + ">" + + (LastChunkEndsWithNewline ? "\n" : "")); + } + +private: + markup::Paragraph &Out; + const comments::CommandTraits &Traits; + + /// If true, the next leading space after a new line is trimmed. + bool LastChunkEndsWithNewline = false; +}; + +class BlockCommentToMarkupDocument + : public comments::ConstCommentVisitor<BlockCommentToMarkupDocument> { +public: + BlockCommentToMarkupDocument(markup::Document &Out, + const comments::CommandTraits &Traits) + : Out(Out), Traits(Traits) {} + + void visitBlockCommandComment(const comments::BlockCommandComment *B) { + + switch (B->getCommandID()) { + case comments::CommandTraits::KCI_arg: + case comments::CommandTraits::KCI_li: + // \li and \arg are special cases, they are used to create a list item. + // In markdown it is a bullet list. + ParagraphToMarkupDocument(Out.addBulletList().addItem().addParagraph(), + Traits) + .visit(B->getParagraph()); + break; + default: { + // Some commands have arguments, like \throws. + // The arguments are not part of the paragraph. + // We need reconstruct them here. + std::string ArgText; + for (unsigned I = 0; I < B->getNumArgs(); ++I) { + if (!ArgText.empty()) + ArgText += " "; + ArgText += B->getArgText(I); + } + auto &P = Out.addParagraph(); + commandToMarkup(P, B->getCommandName(Traits), B->getCommandMarker(), + ArgText); + if (B->getParagraph() && !B->getParagraph()->isWhitespace()) { + // For commands with arguments, the paragraph starts after the first + // space. Therefore we need to append a space manually in this case. + if (!ArgText.empty()) + P.appendSpace(); + ParagraphToMarkupDocument(P, Traits).visit(B->getParagraph()); + } + } + } + } + + void visitVerbatimBlockComment(const comments::VerbatimBlockComment *VB) { + commandToMarkup(Out.addParagraph(), VB->getCommandName(Traits), + VB->getCommandMarker(), ""); + + std::string VerbatimText; + + for (const auto *LI = VB->child_begin(); LI != VB->child_end(); ++LI) { + if (const auto *Line = cast<comments::VerbatimBlockLineComment>(*LI)) { + VerbatimText += Line->getText().str() + "\n"; + } + } + + Out.addCodeBlock(VerbatimText, ""); + + commandToMarkup(Out.addParagraph(), VB->getCloseName(), + VB->getCommandMarker(), ""); + } + + void visitVerbatimLineComment(const comments::VerbatimLineComment *VL) { + auto &P = Out.addParagraph(); + commandToMarkup(P, VL->getCommandName(Traits), VL->getCommandMarker(), ""); + P.appendSpace().appendCode(VL->getText().str(), true).appendSpace(); + } + +private: + markup::Document &Out; + const comments::CommandTraits &Traits; + StringRef CommentEscapeMarker; +}; + +void SymbolDocCommentVisitor::parameterDocToMarkup(StringRef ParamName, + markup::Paragraph &Out) { + if (ParamName.empty()) + return; + + if (const auto *P = Parameters.lookup(ParamName)) { + ParagraphToMarkupDocument(Out, Traits).visit(P->getParagraph()); + } +} + +void SymbolDocCommentVisitor::docToMarkup(markup::Document &Out) { + for (unsigned I = 0; I < CommentPartIndex; ++I) { + if (const auto *BC = BlockCommands.lookup(I)) { + BlockCommentToMarkupDocument(Out, Traits).visit(BC); + } else if (const auto *P = FreeParagraphs.lookup(I)) { + ParagraphToMarkupDocument(Out.addParagraph(), Traits).visit(P); + } + } +} + +} // namespace clangd +} // namespace clang diff --git a/clang-tools-extra/clangd/SymbolDocumentation.h b/clang-tools-extra/clangd/SymbolDocumentation.h new file mode 100644 index 0000000000000..f1ab349858398 --- /dev/null +++ b/clang-tools-extra/clangd/SymbolDocumentation.h @@ -0,0 +1,140 @@ +//===--- SymbolDocumentation.h ==---------------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Class to parse doxygen comments into a flat structure for consumption +// in e.g. Hover and Code Completion +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_SYMBOLDOCUMENTATION_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SYMBOLDOCUMENTATION_H + +#include "support/Markup.h" +#include "clang/AST/Comment.h" +#include "clang/AST/CommentLexer.h" +#include "clang/AST/CommentParser.h" +#include "clang/AST/CommentSema.h" +#include "clang/AST/CommentVisitor.h" +#include "clang/Basic/SourceManager.h" +#include <string> + +namespace clang { +namespace clangd { + +class SymbolDocCommentVisitor + : public comments::ConstCommentVisitor<SymbolDocCommentVisitor> { +public: + SymbolDocCommentVisitor(llvm::StringRef Documentation, + const CommentOptions &CommentOpts) + : Traits(Allocator, CommentOpts), Allocator() { + + if (Documentation.empty()) + return; + + CommentWithMarkers.reserve(Documentation.size() + + Documentation.count('\n') * 3); + + // The comment lexer expects doxygen markers, so add them back. + // We need to use the /// style doxygen markers because the comment could + // contain the closing the closing tag "*/" of a C Style "/** */" comment + // which would break the parsing if we would just enclose the comment text + // with "/** */". + CommentWithMarkers = "///"; + bool NewLine = true; + for (char C : Documentation) { + if (C == '\n') { + CommentWithMarkers += "\n///"; + NewLine = true; + } else { + if (NewLine && (C == '<')) { + // A comment line starting with '///<' is treated as a doxygen + // comment. Therefore add a space to separate the '<' from the comment + // marker. This allows to parse html tags at the beginning of a line + // and the escape marker prevents adding the artificial space in the + // markup documentation. The extra space will not be rendered, since + // we render it as markdown. + CommentWithMarkers += ' '; + } + CommentWithMarkers += C; + NewLine = false; + } + } + SourceManagerForFile SourceMgrForFile("mock_file.cpp", CommentWithMarkers); + + SourceManager &SourceMgr = SourceMgrForFile.get(); + // The doxygen Sema requires a Diagostics consumer, since it reports + // warnings e.g. when parameters are not documented correctly. These + // warnings are not relevant for us, so we can ignore them. + SourceMgr.getDiagnostics().setClient(new IgnoringDiagConsumer); + + comments::Sema S(Allocator, SourceMgr, SourceMgr.getDiagnostics(), Traits, + /*PP=*/nullptr); + comments::Lexer L(Allocator, SourceMgr.getDiagnostics(), Traits, + SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID()), + CommentWithMarkers.data(), + CommentWithMarkers.data() + CommentWithMarkers.size()); + comments::Parser P(L, S, Allocator, SourceMgr, SourceMgr.getDiagnostics(), + Traits); + comments::FullComment *FC = P.parseFullComment(); + + if (FC) { + for (auto *Block : FC->getBlocks()) { + visit(Block); + } + } + } + + bool isParameterDocumented(StringRef ParamName) const { + return Parameters.contains(ParamName); + } + + void parameterDocToMarkup(StringRef ParamName, markup::Paragraph &Out); + + void docToMarkup(markup::Document &Out); + + void visitBlockCommandComment(const comments::BlockCommandComment *B) { + BlockCommands[CommentPartIndex] = std::move(B); + CommentPartIndex++; + } + + void visitParagraphComment(const comments::ParagraphComment *P) { + FreeParagraphs[CommentPartIndex] = std::move(P); + CommentPartIndex++; + } + + void visitParamCommandComment(const comments::ParamCommandComment *P) { + Parameters[P->getParamNameAsWritten()] = std::move(P); + } + +private: + comments::CommandTraits Traits; + llvm::BumpPtrAllocator Allocator; + std::string CommentWithMarkers; + + /// Index to keep track of the order of the comments. + /// We want to rearange some commands like \\param. + /// This index allows us to keep the order of the other comment parts. + unsigned CommentPartIndex = 0; + + /// Parsed paragaph(s) of the "param" comamnd(s) + llvm::SmallDenseMap<StringRef, const comments::ParamCommandComment *> + Parameters; + + /// All the block commands. + llvm::SmallDenseMap<unsigned, const comments::BlockCommandComment *> + BlockCommands; + + /// All "free" text paragraphs. + llvm::SmallDenseMap<unsigned, const comments::ParagraphComment *> + FreeParagraphs; +}; + +} // namespace clangd +} // namespace clang + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_SYMBOLDOCUMENTATION_H diff --git a/clang-tools-extra/clangd/support/Markup.cpp b/clang-tools-extra/clangd/support/Markup.cpp index a13083026f26b..152863191dad1 100644 --- a/clang-tools-extra/clangd/support/Markup.cpp +++ b/clang-tools-extra/clangd/support/Markup.cpp @@ -363,7 +363,12 @@ class CodeBlock : public Block { void renderMarkdown(llvm::raw_ostream &OS) const override { std::string Marker = getMarkerForCodeBlock(Contents); // No need to pad from previous blocks, as they should end with a new line. - OS << Marker << Language << '\n' << Contents << '\n' << Marker << '\n'; + OS << Marker << Language << '\n' << Contents; + if (Contents.back() != '\n') + OS << '\n'; + // Always end with an empty line to separate code blocks from following + // paragraphs. + OS << Marker << "\n\n"; } void renderPlainText(llvm::raw_ostream &OS) const override { diff --git a/clang-tools-extra/clangd/unittests/CMakeLists.txt b/clang-tools-extra/clangd/unittests/CMakeLists.txt index dffdcd5d014ca..bc457a8241fa7 100644 --- a/clang-tools-extra/clangd/unittests/CMakeLists.txt +++ b/clang-tools-extra/clangd/unittests/CMakeLists.txt @@ -92,6 +92,7 @@ add_unittest(ClangdUnitTests ClangdTests SourceCodeTests.cpp StdLibTests.cpp SymbolCollectorTests.cpp + SymbolDocumentationTests.cpp SymbolInfoTests.cpp SyncAPI.cpp TUSchedulerTests.cpp diff --git a/clang-tools-extra/clangd/unittests/HoverTests.cpp b/clang-tools-extra/clangd/unittests/HoverTests.cpp index 12d260db7ea11..fb6f49c987d46 100644 --- a/clang-tools-extra/clangd/unittests/HoverTests.cpp +++ b/clang-tools-extra/clangd/unittests/HoverTests.cpp @@ -3762,6 +3762,127 @@ provides Foo, Bar, Baz, Foobar, Qux and 1 more)"}}; } } +TEST(Hover, PresentDocumentation) { + struct { + const std::function<void(HoverInfo &)> Builder; + llvm::StringRef ExpectedRender; + } Cases[] = { + {[](HoverInfo &HI) { + HI.Kind = index::SymbolKind::Function; + HI.Documentation = "@brief brief doc\n\n" + "longer doc"; + HI.Definition = "void foo()"; + HI.Name = "foo"; + }, + R"(### function `foo` + +--- +**@brief** brief doc + +longer doc + +--- +```cpp +void foo() +```)"}, + {[](HoverInfo &HI) { + HI.Kind = index::SymbolKind::Function; + HI.Documentation = "@brief brief doc\n\n" + "longer doc"; + HI.Definition = "int foo()"; + HI.ReturnType = "int"; + HI.Name = "foo"; + }, + R"(### function `foo` + +--- +→ `int` + +**@brief** brief doc + +longer doc + +--- +```cpp +int foo() +```)"}, + {[](HoverInfo &HI) { + HI.Kind = index::SymbolKind::Function; + HI.Documentation = "@brief brief doc\n\n" + "longer doc\n@param a this is a param\n@return it " + "returns something"; + HI.Definition = "int foo(int a)"; + HI.ReturnType = "int"; + HI.Name = "foo"; + HI.Parameters.emplace(); + HI.Parameters->emplace_back(); + HI.Parameters->back().Type = "int"; + HI.Parameters->back().Name = "a"; + }, + R"(### function `foo` + +--- +→ `int` + +Parameters: + +- `int a` - this is a param + +**@brief** brief doc + +longer doc + +**@return** it returns something + +--- +```cpp +int foo(int a) +```)"}, + {[](HoverInfo &HI) { + HI.Kind = index::SymbolKind::Function; + HI.Documentation = "@brief brief doc\n\n" + "longer doc\n@param a this is a param\n@param b " + "does not exist\n@return it returns something"; + HI.Definition = "int foo(int a)"; + HI.ReturnType = "int"; + HI.Name = "foo"; + HI.Parameters.emplace(); + HI.Parameters->emplace_back(); + HI.Parameters->back().Type = "int"; + HI.Parameters->back().Name = "a"; + }, + R"(### function `foo` + +--- +→ `int` + +Parameters: + +- `int a` - this is a param + +**@brief** brief doc + +longer doc + +**@return** it returns something + +--- +```cpp +int foo(int a) +```)"}, + }; + + for (const auto &C : Cases) { + HoverInfo HI; + C.Builder(HI); + Config Cfg; + Cfg.Hover.ShowAKA = true; + Cfg.Documentation.CommentFormat = Config::CommentFormatPolicy::Doxygen; + WithContextValue WithCfg(Config::Key, std::move(Cfg)); + EXPECT_EQ(HI.present(MarkupKind::Markdown), C.ExpectedRender); + } +} + TEST(Hover, ParseDocumentation) { struct Case { llvm::StringRef Documentation; @@ -4339,6 +4460,114 @@ constexpr u64 pow_with_mod(u64 a, u64 b, u64 p) { EXPECT_TRUE(H->Value); EXPECT_TRUE(H->Type); } + +TEST(Hover, FunctionParameters) { + struct { + const char *const Code; + const std::function<void(HoverInfo &)> ExpectedBuilder; + std::string ExpectedRender; + } Cases[] = { + {R"cpp(/// Function doc + void foo(int [[^a]]); + )cpp", + [](HoverInfo &HI) { + HI.Name = "a"; + HI.Kind = index::SymbolKind::Parameter; + HI.NamespaceScope = ""; + HI.LocalScope = "foo::"; + HI.Type = "int"; + HI.Definition = "int a"; + HI.Documentation = "Function doc"; + }, + "### param `a`\n\n---\nType: `int`\n\n---\n```cpp\n// In foo\nint " + "a\n```"}, + {R"cpp(/// Function doc + /// @param a this is doc for a + void foo(int [[^a]]); + )cpp", + [](HoverInfo &HI) { + HI.Name = "a"; + HI.Kind = index::SymbolKind::Parameter; + HI.NamespaceScope = ""; + HI.LocalScope = "foo::"; + HI.Type = "int"; + HI.Definition = "int a"; + HI.Documentation = "Function doc\n @param a this is doc for a"; + }, + "### param `a`\n\n---\nType: `int`\n\n this is doc for " + "a\n\n---\n```cpp\n// In foo\nint a\n```"}, + {R"cpp(/// Function doc + /// @param b this is doc for b + void foo(int [[^a]], int b); + )cpp", + [](HoverInfo &HI) { + HI.Name = "a"; + HI.Kind = index::SymbolKind::Parameter; + HI.NamespaceScope = ""; + HI.LocalScope = "foo::"; + HI.Type = "int"; + HI.Definition = "int a"; + HI.Documentation = "Function doc\n @param b this is doc for b"; + }, + "### param `a`\n\n---\nType: `int`\n\n---\n```cpp\n// In foo\nint " + "a\n```"}, + {R"cpp(/// Function doc + /// @param b this is doc for \p b + void foo(int a, int [[^b]]); + )cpp", + [](HoverInfo &HI) { + HI.Name = "b"; + HI.Kind = index::SymbolKind::Parameter; + HI.NamespaceScope = ""; + HI.LocalScope = "foo::"; + HI.Type = "int"; + HI.Definition = "int b"; + HI.Documentation = "Function doc\n @param b this is doc for \\p b"; + }, + "### param `b`\n\n---\nType: `int`\n\n this is doc for " + "`b`\n\n---\n```cpp\n// In foo\nint b\n```"}, + }; + + // Create a tiny index, so tests above can verify documentation is fetched. + Symbol IndexSym = func("indexSymbol"); + IndexSym.Documentation = "comment from index"; + SymbolSlab::Builder Symbols; + Symbols.insert(IndexSym); + auto Index = + MemIndex::build(std::move(Symbols).build(), RefSlab(), RelationSlab()); + + for (const auto &Case : Cases) { + SCOPED_TRACE(Case.Code); + + Annotations T(Case.Code); + TestTU TU = TestTU::withCode(T.code()); + auto AST = TU.build(); + Config Cfg; + Cfg.Hover.ShowAKA = true; + Cfg.Documentation.CommentFormat = Config::CommentFormatPolicy::Doxygen; + WithContextValue WithCfg(Config::Key, std::move(Cfg)); + auto H = getHover(AST, T.point(), format::getLLVMStyle(), Index.get()); + ASSERT_TRUE(H); + HoverInfo Expected; + Expected.SymRange = T.range(); + Case.ExpectedBuilder(Expected); + + EXPECT_EQ(H->present(MarkupKind::Markdown), Case.ExpectedRender); + EXPECT_EQ(H->NamespaceScope, Expected.NamespaceScope); + EXPECT_EQ(H->LocalScope, Expected.LocalScope); + EXPECT_EQ(H->Name, Expected.Name); + EXPECT_EQ(H->Kind, Expected.Kind); + EXPECT_EQ(H->Documentation, Expected.Documentation); + EXPECT_EQ(H->Definition, Expected.Definition); + EXPECT_EQ(H->Type, Expected.Type); + EXPECT_EQ(H->ReturnType, Expected.ReturnType); + EXPECT_EQ(H->Parameters, Expected.Parameters); + EXPECT_EQ(H->TemplateParameters, Expected.TemplateParameters); + EXPECT_EQ(H->SymRange, Expected.SymRange); + EXPECT_EQ(H->Value, Expected.Value); + } +} + } // namespace } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/unittests/SymbolDocumentationTests.cpp b/clang-tools-extra/clangd/unittests/SymbolDocumentationTests.cpp new file mode 100644 index 0000000000000..69eb13b2142d2 --- /dev/null +++ b/clang-tools-extra/clangd/unittests/SymbolDocumentationTests.cpp @@ -0,0 +1,161 @@ +//===-- SymbolDocumentationTests.cpp --------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#include "SymbolDocumentation.h" + +#include "support/Markup.h" +#include "clang/Basic/CommentOptions.h" +#include "llvm/ADT/StringRef.h" +#include "gtest/gtest.h" + +namespace clang { +namespace clangd { + +TEST(SymbolDocumentation, Parse) { + + CommentOptions CommentOpts; + + struct Case { + llvm::StringRef Documentation; + llvm::StringRef ExpectedRenderEscapedMarkdown; + llvm::StringRef ExpectedRenderMarkdown; + llvm::StringRef ExpectedRenderPlainText; + } Cases[] = { + { + "foo bar", + "foo bar", + "foo bar", + "foo bar", + }, + { + "foo\nbar\n", + "foo\nbar", + "foo\nbar", + "foo bar", + }, + { + "foo\n\nbar\n", + "foo\n\nbar", + "foo\n\nbar", + "foo\n\nbar", + }, + { + "foo \\p bar baz", + "foo `bar` baz", + "foo `bar` baz", + "foo bar baz", + }, + { + "foo \\e bar baz", + "foo \\*bar\\* baz", + "foo *bar* baz", + "foo *bar* baz", + }, + { + "foo \\b bar baz", + "foo \\*\\*bar\\*\\* baz", + "foo **bar** baz", + "foo **bar** baz", + }, + { + "foo \\ref bar baz", + "foo \\*\\*\\\\ref\\*\\* \\*bar\\* baz", + "foo **\\ref** *bar* baz", + "foo **\\ref** *bar* baz", + }, + { + "foo @ref bar baz", + "foo \\*\\*@ref\\*\\* \\*bar\\* baz", + "foo **@ref** *bar* baz", + "foo **@ref** *bar* baz", + }, + { + "\\brief this is a \\n\nbrief description", + "\\*\\*\\\\brief\\*\\* this is a \nbrief description", + "**\\brief** this is a \nbrief description", + "**\\brief** this is a\nbrief description", + }, + { + "\\throw exception foo", + "\\*\\*\\\\throw\\*\\* \\*exception\\* foo", + "**\\throw** *exception* foo", + "**\\throw** *exception* foo", + }, + { + "\\brief this is a brief description\n\n\\li item 1\n\\li item " + "2\n\\arg item 3", + "\\*\\*\\\\brief\\*\\* this is a brief description\n\n- item 1\n\n- " + "item " + "2\n\n- " + "item 3", + "**\\brief** this is a brief description\n\n- item 1\n\n- item " + "2\n\n- " + "item 3", + "**\\brief** this is a brief description\n\n- item 1\n\n- item " + "2\n\n- " + "item 3", + }, + { + "\\defgroup mygroup this is a group\nthis is not a group description", + "\\*\\*@defgroup\\*\\* `mygroup this is a group`\n\nthis is not a " + "group " + "description", + "**@defgroup** `mygroup this is a group`\n\nthis is not a group " + "description", + "**@defgroup** `mygroup this is a group`\n\nthis is not a group " + "description", + }, + { + "\\verbatim\nthis is a\nverbatim block containing\nsome verbatim " + "text\n\\endverbatim", + "\\*\\*@verbatim\\*\\*\n\n```\nthis is a\nverbatim block " + "containing\nsome " + "verbatim text\n```\n\n\\*\\*@endverbatim\\*\\*", + "**@verbatim**\n\n```\nthis is a\nverbatim block containing\nsome " + "verbatim text\n```\n\n**@endverbatim**", + "**@verbatim**\n\nthis is a\nverbatim block containing\nsome " + "verbatim text\n\n**@endverbatim**", + }, + { + "@param foo this is a parameter\n@param bar this is another " + "parameter", + "", + "", + "", + }, + { + "@brief brief docs\n\n@param foo this is a parameter\n\nMore " + "description\ndocumentation", + "\\*\\*@brief\\*\\* brief docs\n\nMore description\ndocumentation", + "**@brief** brief docs\n\nMore description\ndocumentation", + "**@brief** brief docs\n\nMore description documentation", + }, + { + "<b>this is a bold text</b>\nnormal text\n<i>this is an italic " + "text</i>\n<code>this is a code block</code>", + "\\<b>this is a bold text\\</b>\nnormal text\n\\<i>this is an italic " + "text\\</i>\n\\<code>this is a code block\\</code>", + "\\<b>this is a bold text\\</b>\nnormal text\n\\<i>this is an italic " + "text\\</i>\n\\<code>this is a code block\\</code>", + "<b>this is a bold text</b> normal text <i>this is an italic " + "text</i> <code>this is a code block</code>", + }, + }; + for (const auto &C : Cases) { + markup::Document Doc; + SymbolDocCommentVisitor SymbolDoc(C.Documentation, CommentOpts); + + SymbolDoc.docToMarkup(Doc); + + EXPECT_EQ(Doc.asPlainText(), C.ExpectedRenderPlainText); + EXPECT_EQ(Doc.asMarkdown(), C.ExpectedRenderMarkdown); + EXPECT_EQ(Doc.asEscapedMarkdown(), C.ExpectedRenderEscapedMarkdown); + } +} + +} // namespace clangd +} // namespace clang diff --git a/clang-tools-extra/clangd/unittests/support/MarkupTests.cpp b/clang-tools-extra/clangd/unittests/support/MarkupTests.cpp index 482f230fb86fe..9c17db067f398 100644 --- a/clang-tools-extra/clangd/unittests/support/MarkupTests.cpp +++ b/clang-tools-extra/clangd/unittests/support/MarkupTests.cpp @@ -463,6 +463,7 @@ TEST(Document, Separators) { ```cpp test ``` + bar)md"; EXPECT_EQ(D.asEscapedMarkdown(), ExpectedMarkdown); EXPECT_EQ(D.asMarkdown(), ExpectedMarkdown); @@ -559,6 +560,7 @@ foo bar baz ``` + ```cpp foo ```)md"; diff --git a/clang/include/clang/AST/Comment.h b/clang/include/clang/AST/Comment.h index dd9906727293f..42686ff24076a 100644 --- a/clang/include/clang/AST/Comment.h +++ b/clang/include/clang/AST/Comment.h @@ -19,6 +19,7 @@ #include "clang/Basic/SourceLocation.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Support/Compiler.h" namespace clang { class Decl; @@ -119,6 +120,11 @@ class Comment { LLVM_PREFERRED_TYPE(CommandTraits::KnownCommandIDs) unsigned CommandID : CommandInfo::NumCommandIDBits; + + /// Describes the syntax that was used in a documentation command. + /// Contains values from CommandMarkerKind enum. + LLVM_PREFERRED_TYPE(CommandMarkerKind) + unsigned CommandMarker : 1; }; enum { NumInlineCommandCommentBits = NumInlineContentCommentBits + 3 + CommandInfo::NumCommandIDBits }; @@ -347,6 +353,16 @@ class InlineCommandComment : public InlineContentComment { InlineCommandCommentBits.RenderKind = llvm::to_underlying(RK); InlineCommandCommentBits.CommandID = CommandID; } + InlineCommandComment(SourceLocation LocBegin, SourceLocation LocEnd, + unsigned CommandID, InlineCommandRenderKind RK, + CommandMarkerKind CommandMarker, ArrayRef<Argument> Args) + : InlineContentComment(CommentKind::InlineCommandComment, LocBegin, + LocEnd), + Args(Args) { + InlineCommandCommentBits.RenderKind = llvm::to_underlying(RK); + InlineCommandCommentBits.CommandID = CommandID; + InlineCommandCommentBits.CommandMarker = llvm::to_underlying(CommandMarker); + } static bool classof(const Comment *C) { return C->getCommentKind() == CommentKind::InlineCommandComment; @@ -384,6 +400,11 @@ class InlineCommandComment : public InlineContentComment { SourceRange getArgRange(unsigned Idx) const { return Args[Idx].Range; } + + CommandMarkerKind getCommandMarker() const LLVM_READONLY { + return static_cast<CommandMarkerKind>( + InlineCommandCommentBits.CommandMarker); + } }; /// Abstract class for opening and closing HTML tags. HTML tags are always diff --git a/clang/include/clang/AST/CommentSema.h b/clang/include/clang/AST/CommentSema.h index 916d7945329c5..3169e2b0d86b9 100644 --- a/clang/include/clang/AST/CommentSema.h +++ b/clang/include/clang/AST/CommentSema.h @@ -131,6 +131,7 @@ class Sema { InlineCommandComment *actOnInlineCommand(SourceLocation CommandLocBegin, SourceLocation CommandLocEnd, unsigned CommandID, + CommandMarkerKind CommandMarker, ArrayRef<Comment::Argument> Args); InlineContentComment *actOnUnknownCommand(SourceLocation LocBegin, diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp index e61846d241915..2e5821a8e4436 100644 --- a/clang/lib/AST/CommentParser.cpp +++ b/clang/lib/AST/CommentParser.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "clang/AST/CommentParser.h" +#include "clang/AST/Comment.h" #include "clang/AST/CommentCommandTraits.h" #include "clang/AST/CommentSema.h" #include "clang/Basic/CharInfo.h" @@ -569,6 +570,8 @@ BlockCommandComment *Parser::parseBlockCommand() { InlineCommandComment *Parser::parseInlineCommand() { assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command)); + CommandMarkerKind CMK = + Tok.is(tok::backslash_command) ? CMK_Backslash : CMK_At; const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID()); const Token CommandTok = Tok; @@ -580,7 +583,7 @@ InlineCommandComment *Parser::parseInlineCommand() { InlineCommandComment *IC = S.actOnInlineCommand( CommandTok.getLocation(), CommandTok.getEndLocation(), - CommandTok.getCommandID(), Args); + CommandTok.getCommandID(), CMK, Args); if (Args.size() < Info->NumArgs) { Diag(CommandTok.getEndLocation().getLocWithOffset(1), diff --git a/clang/lib/AST/CommentSema.cpp b/clang/lib/AST/CommentSema.cpp index 88520d7940e34..c02983b03163f 100644 --- a/clang/lib/AST/CommentSema.cpp +++ b/clang/lib/AST/CommentSema.cpp @@ -363,12 +363,13 @@ void Sema::actOnTParamCommandFinish(TParamCommandComment *Command, InlineCommandComment * Sema::actOnInlineCommand(SourceLocation CommandLocBegin, SourceLocation CommandLocEnd, unsigned CommandID, + CommandMarkerKind CommandMarker, ArrayRef<Comment::Argument> Args) { StringRef CommandName = Traits.getCommandInfo(CommandID)->Name; - return new (Allocator) - InlineCommandComment(CommandLocBegin, CommandLocEnd, CommandID, - getInlineCommandRenderKind(CommandName), Args); + return new (Allocator) InlineCommandComment( + CommandLocBegin, CommandLocEnd, CommandID, + getInlineCommandRenderKind(CommandName), CommandMarker, Args); } InlineContentComment *Sema::actOnUnknownCommand(SourceLocation LocBegin, diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn index b609d4a7462fb..f8c4838ab7ee3 100644 --- a/llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn +++ b/llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn @@ -122,6 +122,7 @@ static_library("clangd") { "SemanticHighlighting.cpp", "SemanticSelection.cpp", "SourceCode.cpp", + "SymbolDocumentation.cpp", "SystemIncludeExtractor.cpp", "TUScheduler.cpp", "TidyProvider.cpp", _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits