llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-clang Author: None (tcottin) <details> <summary>Changes</summary> Followup work of #<!-- -->140498 to continue the work on clangd/clangd#<!-- -->529 Introduce the use of the Clang doxygen parser to parse the documentation of hovered code. - ASTContext independent doxygen parsing - Parsing doxygen commands to markdown for hover information Note: after this PR I have planned another patch to rearrange the information shown in the hover info. This PR is just for the basic introduction of doxygen parsing for hover information. --- Patch is 44.39 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/150790.diff 16 Files Affected: - (modified) clang-tools-extra/clangd/CMakeLists.txt (+1) - (modified) clang-tools-extra/clangd/CodeCompletionStrings.cpp (+14-1) - (modified) clang-tools-extra/clangd/Hover.cpp (+177-10) - (modified) clang-tools-extra/clangd/Hover.h (+10-3) - (added) clang-tools-extra/clangd/SymbolDocumentation.cpp (+221) - (added) clang-tools-extra/clangd/SymbolDocumentation.h (+140) - (modified) clang-tools-extra/clangd/support/Markup.cpp (+6-1) - (modified) clang-tools-extra/clangd/unittests/CMakeLists.txt (+1) - (modified) clang-tools-extra/clangd/unittests/HoverTests.cpp (+229) - (added) clang-tools-extra/clangd/unittests/SymbolDocumentationTests.cpp (+161) - (modified) clang-tools-extra/clangd/unittests/support/MarkupTests.cpp (+2) - (modified) clang/include/clang/AST/Comment.h (+21) - (modified) clang/include/clang/AST/CommentSema.h (+1) - (modified) clang/lib/AST/CommentParser.cpp (+4-1) - (modified) clang/lib/AST/CommentSema.cpp (+4-3) - (modified) llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn (+1) ``````````diff diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt index a1e9da41b4b32..06920a97ddc88 100644 --- a/clang-tools-extra/clangd/CMakeLists.txt +++ b/clang-tools-extra/clangd/CMakeLists.txt @@ -108,6 +108,7 @@ add_clang_library(clangDaemon STATIC SemanticHighlighting.cpp SemanticSelection.cpp SourceCode.cpp + SymbolDocumentation.cpp SystemIncludeExtractor.cpp TidyProvider.cpp TUScheduler.cpp diff --git a/clang-tools-extra/clangd/CodeCompletionStrings.cpp b/clang-tools-extra/clangd/CodeCompletionStrings.cpp index 9b4442b0bb76f..196a1624e1c04 100644 --- a/clang-tools-extra/clangd/CodeCompletionStrings.cpp +++ b/clang-tools-extra/clangd/CodeCompletionStrings.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "CodeCompletionStrings.h" +#include "Config.h" #include "clang-c/Index.h" #include "clang/AST/ASTContext.h" #include "clang/AST/RawCommentList.h" @@ -100,7 +101,19 @@ std::string getDeclComment(const ASTContext &Ctx, const NamedDecl &Decl) { // the comments for namespaces. return ""; } - const RawComment *RC = getCompletionComment(Ctx, &Decl); + + const RawComment *RC = nullptr; + const Config &Cfg = Config::current(); + + if (Cfg.Documentation.CommentFormat == Config::CommentFormatPolicy::Doxygen && + isa<ParmVarDecl>(Decl)) { + // Parameters are documented in the function comment. + if (const auto *FD = dyn_cast<FunctionDecl>(Decl.getDeclContext())) + RC = getCompletionComment(Ctx, FD); + } else { + RC = getCompletionComment(Ctx, &Decl); + } + if (!RC) return ""; // Sanity check that the comment does not come from the PCH. We choose to not diff --git a/clang-tools-extra/clangd/Hover.cpp b/clang-tools-extra/clangd/Hover.cpp index 1e0718d673260..63fdc7c24a7a8 100644 --- a/clang-tools-extra/clangd/Hover.cpp +++ b/clang-tools-extra/clangd/Hover.cpp @@ -18,6 +18,7 @@ #include "Protocol.h" #include "Selection.h" #include "SourceCode.h" +#include "SymbolDocumentation.h" #include "clang-include-cleaner/Analysis.h" #include "clang-include-cleaner/IncludeSpeller.h" #include "clang-include-cleaner/Types.h" @@ -41,6 +42,7 @@ #include "clang/AST/Type.h" #include "clang/Basic/CharInfo.h" #include "clang/Basic/LLVM.h" +#include "clang/Basic/LangOptions.h" #include "clang/Basic/SourceLocation.h" #include "clang/Basic/SourceManager.h" #include "clang/Basic/Specifiers.h" @@ -627,6 +629,9 @@ HoverInfo getHoverContents(const NamedDecl *D, const PrintingPolicy &PP, HI.Name = printName(Ctx, *D); const auto *CommentD = getDeclForComment(D); HI.Documentation = getDeclComment(Ctx, *CommentD); + // safe the language options to be able to create the comment::CommandTraits + // to parse the documentation + HI.CommentOpts = D->getASTContext().getLangOpts().CommentOpts; enhanceFromIndex(HI, *CommentD, Index); if (HI.Documentation.empty()) HI.Documentation = synthesizeDocumentation(D); @@ -1388,9 +1393,170 @@ static std::string formatOffset(uint64_t OffsetInBits) { return Offset; } -markup::Document HoverInfo::present() const { +markup::Document HoverInfo::presentDoxygen() const { + // NOTE: this function is currently almost identical to presentDefault(). + // This is to have a minimal change when introducing the doxygen parser. + // This function will be changed when rearranging the output for doxygen + // parsed documentation. + markup::Document Output; + // Header contains a text of the form: + // variable `var` + // + // class `X` + // + // function `foo` + // + // expression + // + // Note that we are making use of a level-3 heading because VSCode renders + // level 1 and 2 headers in a huge font, see + // https://github.com/microsoft/vscode/issues/88417 for details. + markup::Paragraph &Header = Output.addHeading(3); + if (Kind != index::SymbolKind::Unknown) + Header.appendText(index::getSymbolKindString(Kind)).appendSpace(); + assert(!Name.empty() && "hover triggered on a nameless symbol"); + + Header.appendCode(Name); + + if (!Provider.empty()) { + markup::Paragraph &DI = Output.addParagraph(); + DI.appendText("provided by"); + DI.appendSpace(); + DI.appendCode(Provider); + Output.addRuler(); + } + + // Put a linebreak after header to increase readability. + Output.addRuler(); + // Print Types on their own lines to reduce chances of getting line-wrapped by + // editor, as they might be long. + if (ReturnType) { + // For functions we display signature in a list form, e.g.: + // → `x` + // Parameters: + // - `bool param1` + // - `int param2 = 5` + Output.addParagraph().appendText("→ ").appendCode( + llvm::to_string(*ReturnType)); + } + + SymbolDocCommentVisitor SymbolDoc(Documentation, CommentOpts); + + if (Parameters && !Parameters->empty()) { + Output.addParagraph().appendText("Parameters:"); + markup::BulletList &L = Output.addBulletList(); + for (const auto &Param : *Parameters) { + markup::Paragraph &P = L.addItem().addParagraph(); + P.appendCode(llvm::to_string(Param)); + + if (SymbolDoc.isParameterDocumented(llvm::to_string(Param.Name))) { + P.appendText(" -"); + SymbolDoc.parameterDocToMarkup(llvm::to_string(Param.Name), P); + } + } + } + // Don't print Type after Parameters or ReturnType as this will just duplicate + // the information + if (Type && !ReturnType && !Parameters) + Output.addParagraph().appendText("Type: ").appendCode( + llvm::to_string(*Type)); + + if (Value) { + markup::Paragraph &P = Output.addParagraph(); + P.appendText("Value = "); + P.appendCode(*Value); + } + + if (Offset) + Output.addParagraph().appendText("Offset: " + formatOffset(*Offset)); + if (Size) { + auto &P = Output.addParagraph().appendText("Size: " + formatSize(*Size)); + if (Padding && *Padding != 0) { + P.appendText( + llvm::formatv(" (+{0} padding)", formatSize(*Padding)).str()); + } + if (Align) + P.appendText(", alignment " + formatSize(*Align)); + } + + if (CalleeArgInfo) { + assert(CallPassType); + std::string Buffer; + llvm::raw_string_ostream OS(Buffer); + OS << "Passed "; + if (CallPassType->PassBy != HoverInfo::PassType::Value) { + OS << "by "; + if (CallPassType->PassBy == HoverInfo::PassType::ConstRef) + OS << "const "; + OS << "reference "; + } + if (CalleeArgInfo->Name) + OS << "as " << CalleeArgInfo->Name; + else if (CallPassType->PassBy == HoverInfo::PassType::Value) + OS << "by value"; + if (CallPassType->Converted && CalleeArgInfo->Type) + OS << " (converted to " << CalleeArgInfo->Type->Type << ")"; + Output.addParagraph().appendText(OS.str()); + } + if (Kind == index::SymbolKind::Parameter) { + if (SymbolDoc.isParameterDocumented(Name)) + SymbolDoc.parameterDocToMarkup(Name, Output.addParagraph()); + } else + SymbolDoc.docToMarkup(Output); + + if (!Definition.empty()) { + Output.addRuler(); + std::string Buffer; + + if (!Definition.empty()) { + // Append scope comment, dropping trailing "::". + // Note that we don't print anything for global namespace, to not annoy + // non-c++ projects or projects that are not making use of namespaces. + if (!LocalScope.empty()) { + // Container name, e.g. class, method, function. + // We might want to propagate some info about container type to print + // function foo, class X, method X::bar, etc. + Buffer += + "// In " + llvm::StringRef(LocalScope).rtrim(':').str() + '\n'; + } else if (NamespaceScope && !NamespaceScope->empty()) { + Buffer += "// In namespace " + + llvm::StringRef(*NamespaceScope).rtrim(':').str() + '\n'; + } + + if (!AccessSpecifier.empty()) { + Buffer += AccessSpecifier + ": "; + } + + Buffer += Definition; + } + + Output.addCodeBlock(Buffer, DefinitionLanguage); + } + + if (!UsedSymbolNames.empty()) { + Output.addRuler(); + markup::Paragraph &P = Output.addParagraph(); + P.appendText("provides "); + + const std::vector<std::string>::size_type SymbolNamesLimit = 5; + auto Front = llvm::ArrayRef(UsedSymbolNames).take_front(SymbolNamesLimit); + + llvm::interleave( + Front, [&](llvm::StringRef Sym) { P.appendCode(Sym); }, + [&] { P.appendText(", "); }); + if (UsedSymbolNames.size() > Front.size()) { + P.appendText(" and "); + P.appendText(std::to_string(UsedSymbolNames.size() - Front.size())); + P.appendText(" more"); + } + } + return Output; +} + +markup::Document HoverInfo::presentDefault() const { + markup::Document Output; // Header contains a text of the form: // variable `var` // @@ -1538,21 +1704,22 @@ markup::Document HoverInfo::present() const { std::string HoverInfo::present(MarkupKind Kind) const { if (Kind == MarkupKind::Markdown) { const Config &Cfg = Config::current(); - if ((Cfg.Documentation.CommentFormat == - Config::CommentFormatPolicy::Markdown) || - (Cfg.Documentation.CommentFormat == - Config::CommentFormatPolicy::Doxygen)) - // If the user prefers Markdown, we use the present() method to generate - // the Markdown output. - return present().asMarkdown(); + if (Cfg.Documentation.CommentFormat == + Config::CommentFormatPolicy::Markdown) + return presentDefault().asMarkdown(); + if (Cfg.Documentation.CommentFormat == + Config::CommentFormatPolicy::Doxygen) { + std::string T = presentDoxygen().asMarkdown(); + return T; + } if (Cfg.Documentation.CommentFormat == Config::CommentFormatPolicy::PlainText) // If the user prefers plain text, we use the present() method to generate // the plain text output. - return present().asEscapedMarkdown(); + return presentDefault().asEscapedMarkdown(); } - return present().asPlainText(); + return presentDefault().asPlainText(); } // If the backtick at `Offset` starts a probable quoted range, return the range diff --git a/clang-tools-extra/clangd/Hover.h b/clang-tools-extra/clangd/Hover.h index 2f65431bd72de..2578e7a4339d0 100644 --- a/clang-tools-extra/clangd/Hover.h +++ b/clang-tools-extra/clangd/Hover.h @@ -74,6 +74,8 @@ struct HoverInfo { std::optional<Range> SymRange; index::SymbolKind Kind = index::SymbolKind::Unknown; std::string Documentation; + // required to create a comments::CommandTraits object without the ASTContext + CommentOptions CommentOpts; /// Source code containing the definition of the symbol. std::string Definition; const char *DefinitionLanguage = "cpp"; @@ -118,10 +120,15 @@ struct HoverInfo { // alphabetical order. std::vector<std::string> UsedSymbolNames; - /// Produce a user-readable information. - markup::Document present() const; - + /// Produce a user-readable information based on the specified markup kind. std::string present(MarkupKind Kind) const; + +private: + /// Parse and render the hover information as Doxygen documentation. + markup::Document presentDoxygen() const; + + /// Render the hover information as a default documentation. + markup::Document presentDefault() const; }; inline bool operator==(const HoverInfo::PrintedType &LHS, diff --git a/clang-tools-extra/clangd/SymbolDocumentation.cpp b/clang-tools-extra/clangd/SymbolDocumentation.cpp new file mode 100644 index 0000000000000..1c14ccb01fc26 --- /dev/null +++ b/clang-tools-extra/clangd/SymbolDocumentation.cpp @@ -0,0 +1,221 @@ +//===--- SymbolDocumentation.cpp ==-------------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "SymbolDocumentation.h" + +#include "support/Markup.h" +#include "clang/AST/Comment.h" +#include "clang/AST/CommentCommandTraits.h" +#include "clang/AST/CommentVisitor.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringRef.h" + +namespace clang { +namespace clangd { + +void commandToMarkup(markup::Paragraph &Out, StringRef Command, + comments::CommandMarkerKind CommandMarker, + StringRef Args) { + Out.appendBoldText( + (CommandMarker == (comments::CommandMarkerKind::CMK_At) ? "@" : "\\") + + Command.str()); + if (!Args.empty()) { + Out.appendSpace(); + Out.appendEmphasizedText(Args.str()); + } +} + +class ParagraphToMarkupDocument + : public comments::ConstCommentVisitor<ParagraphToMarkupDocument> { +public: + ParagraphToMarkupDocument(markup::Paragraph &Out, + const comments::CommandTraits &Traits) + : Out(Out), Traits(Traits) {} + + void visitParagraphComment(const comments::ParagraphComment *C) { + if (!C) + return; + + for (const auto *Child = C->child_begin(); Child != C->child_end(); + ++Child) { + visit(*Child); + } + } + + void visitTextComment(const comments::TextComment *C) { + // Always trim leading space after a newline. + StringRef Text = C->getText(); + if (LastChunkEndsWithNewline && C->getText().starts_with(' ')) + Text = Text.drop_front(); + + LastChunkEndsWithNewline = C->hasTrailingNewline(); + Out.appendText(Text.str() + (LastChunkEndsWithNewline ? "\n" : "")); + } + + void visitInlineCommandComment(const comments::InlineCommandComment *C) { + + if (C->getNumArgs() > 0) { + std::string ArgText; + for (unsigned I = 0; I < C->getNumArgs(); ++I) { + if (!ArgText.empty()) + ArgText += " "; + ArgText += C->getArgText(I); + } + + switch (C->getRenderKind()) { + case comments::InlineCommandRenderKind::Monospaced: + Out.appendCode(ArgText); + break; + case comments::InlineCommandRenderKind::Bold: + Out.appendBoldText(ArgText); + break; + case comments::InlineCommandRenderKind::Emphasized: + Out.appendEmphasizedText(ArgText); + break; + default: + commandToMarkup(Out, C->getCommandName(Traits), C->getCommandMarker(), + ArgText); + break; + } + } else { + if (C->getCommandName(Traits) == "n") { + // \n is a special case, it is used to create a new line. + Out.appendText(" \n"); + LastChunkEndsWithNewline = true; + return; + } + + commandToMarkup(Out, C->getCommandName(Traits), C->getCommandMarker(), + ""); + } + } + + void visitHTMLStartTagComment(const comments::HTMLStartTagComment *STC) { + std::string TagText = "<" + STC->getTagName().str(); + + for (unsigned I = 0; I < STC->getNumAttrs(); ++I) { + const comments::HTMLStartTagComment::Attribute &Attr = STC->getAttr(I); + TagText += " " + Attr.Name.str() + "=\"" + Attr.Value.str() + "\""; + } + + if (STC->isSelfClosing()) + TagText += " /"; + TagText += ">"; + + LastChunkEndsWithNewline = STC->hasTrailingNewline(); + Out.appendText(TagText + (LastChunkEndsWithNewline ? "\n" : "")); + } + + void visitHTMLEndTagComment(const comments::HTMLEndTagComment *ETC) { + LastChunkEndsWithNewline = ETC->hasTrailingNewline(); + Out.appendText("</" + ETC->getTagName().str() + ">" + + (LastChunkEndsWithNewline ? "\n" : "")); + } + +private: + markup::Paragraph &Out; + const comments::CommandTraits &Traits; + + /// If true, the next leading space after a new line is trimmed. + bool LastChunkEndsWithNewline = false; +}; + +class BlockCommentToMarkupDocument + : public comments::ConstCommentVisitor<BlockCommentToMarkupDocument> { +public: + BlockCommentToMarkupDocument(markup::Document &Out, + const comments::CommandTraits &Traits) + : Out(Out), Traits(Traits) {} + + void visitBlockCommandComment(const comments::BlockCommandComment *B) { + + switch (B->getCommandID()) { + case comments::CommandTraits::KCI_arg: + case comments::CommandTraits::KCI_li: + // \li and \arg are special cases, they are used to create a list item. + // In markdown it is a bullet list. + ParagraphToMarkupDocument(Out.addBulletList().addItem().addParagraph(), + Traits) + .visit(B->getParagraph()); + break; + default: { + // Some commands have arguments, like \throws. + // The arguments are not part of the paragraph. + // We need reconstruct them here. + std::string ArgText; + for (unsigned I = 0; I < B->getNumArgs(); ++I) { + if (!ArgText.empty()) + ArgText += " "; + ArgText += B->getArgText(I); + } + auto &P = Out.addParagraph(); + commandToMarkup(P, B->getCommandName(Traits), B->getCommandMarker(), + ArgText); + if (B->getParagraph() && !B->getParagraph()->isWhitespace()) { + // For commands with arguments, the paragraph starts after the first + // space. Therefore we need to append a space manually in this case. + if (!ArgText.empty()) + P.appendSpace(); + ParagraphToMarkupDocument(P, Traits).visit(B->getParagraph()); + } + } + } + } + + void visitVerbatimBlockComment(const comments::VerbatimBlockComment *VB) { + commandToMarkup(Out.addParagraph(), VB->getCommandName(Traits), + VB->getCommandMarker(), ""); + + std::string VerbatimText; + + for (const auto *LI = VB->child_begin(); LI != VB->child_end(); ++LI) { + if (const auto *Line = cast<comments::VerbatimBlockLineComment>(*LI)) { + VerbatimText += Line->getText().str() + "\n"; + } + } + + Out.addCodeBlock(VerbatimText, ""); + + commandToMarkup(Out.addParagraph(), VB->getCloseName(), + VB->getCommandMarker(), ""); + } + + void visitVerbatimLineComment(const comments::VerbatimLineComment *VL) { + auto &P = Out.addParagraph(); + commandToMarkup(P, VL->getCommandName(Traits), VL->getCommandMarker(), ""); + P.appendSpace().appendCode(VL->getText().str(), true).appendSpace(); + } + +private: + markup::Document &Out; + const comments::CommandTraits &Traits; + StringRef CommentEscapeMarker; +}; + +void SymbolDocCommentVisitor::parameterDocToMarkup(StringRef ParamName, + markup::Paragraph &Out) { + if (ParamName.empty()) + return; + + if (const auto *P = Parameters.lookup(ParamName)) { + ParagraphToMarkupDocument(Out, Traits).visit(P->getParagraph()); + } +} + +void SymbolDocCommentVisitor::docToMarkup(markup::Document &Out) { + for (unsigned I = 0; I < CommentPartIndex; ++I) { + if (const auto *BC = BlockCommands.lookup(I)) { + BlockCommentToMarkupDocument(Out, Traits).visit(BC); + } else if (const auto *P = FreeParagraphs.lookup(I)) { + ParagraphToMarkupDocument(Out.addParagraph(), Traits).visit(P); + } + } +} + +} // namespace clangd +} // namespace clang diff --git a/clang-tools-extra/clangd/SymbolDocumentation.h b/clang-tools-extra/clangd/SymbolDocumentation.h new file mode 100644 index 0000000000000..f1ab349858398 --- /dev/null +++ b/clang-tools-extra/clangd/SymbolDocumentation.h @@ -0,0 +1,140 @@ +//===--- SymbolDocumentation.h ==---------------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Class to parse doxygen comments into a flat structure for consumption +// in e.g. Hover and Code Completion +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_SYMBOLDOCUMENTATION_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SYMBOLDOCUMENTATION_H + +#include "support/Markup.h" +#include "clang/AST/Comment.h" +#include "clang/AST/CommentLexer.h" +#include "clang/AST/CommentParser.h" +#include "clang/AST/CommentSema.h" +#include "clang/AST/CommentVisitor.h" +#include "clang/Basic/SourceManager.h" +... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/150790 _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
