https://github.com/yronglin updated https://github.com/llvm/llvm-project/pull/191004
>From 2531f10a692bab883685be3b9d9f52742f5bb1a2 Mon Sep 17 00:00:00 2001 From: yronglin <[email protected]> Date: Thu, 9 Apr 2026 00:19:55 +0800 Subject: [PATCH 01/14] [C++][Modules][Preprocessor] Clang should not convert a import preprocessing token to contextual keyword if a digraph character following import Signed-off-by: yronglin <[email protected]> --- clang/docs/ReleaseNotes.md | 1 + clang/include/clang/Lex/Lexer.h | 4 ++++ clang/lib/Lex/Lexer.cpp | 26 +++++++++++++++--------- clang/lib/Lex/Preprocessor.cpp | 30 ++++++++++++++++++++++++---- clang/test/CXX/module/cpp.pre/p1.cpp | 26 ++++++++++++++++++++++++ 5 files changed, 74 insertions(+), 13 deletions(-) diff --git a/clang/docs/ReleaseNotes.md b/clang/docs/ReleaseNotes.md index 50e1c4ce88797..d2ec285d19391 100644 --- a/clang/docs/ReleaseNotes.md +++ b/clang/docs/ReleaseNotes.md @@ -319,6 +319,7 @@ features cannot lower the translation-unit ABI level; ### Bug Fixes in This Version +- Fixed incorrect handling of C++ import preprocessing token when a digraph character after import. (#GH190693) - Fixed a constraint comparison bug in partial ordering. (#GH182671) - Fixed a rejected-valid case that used an explicit object parameter in an out-of-line definition of a nested class member. (#GH136472) diff --git a/clang/include/clang/Lex/Lexer.h b/clang/include/clang/Lex/Lexer.h index b042e5fb088fa..cd872cfcebb7e 100644 --- a/clang/include/clang/Lex/Lexer.h +++ b/clang/include/clang/Lex/Lexer.h @@ -740,6 +740,10 @@ class Lexer : public PreprocessorLexer { /// otherwise return P. static const char *SkipEscapedNewLines(const char *P); + /// SkipHorizontalWhitespace - Skip the horizontak whitespace characters and + /// returns the advanced pointer. + static const char *SkipHorizontalWhitespace(const char *Ptr); + /// getCharAndSizeSlowNoWarn - Same as getCharAndSizeSlow, but never emits a /// diagnostic. static SizedChar getCharAndSizeSlowNoWarn(const char *Ptr, diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp index f7d44879a936d..c5e0a8314d185 100644 --- a/clang/lib/Lex/Lexer.cpp +++ b/clang/lib/Lex/Lexer.cpp @@ -1372,6 +1372,18 @@ const char *Lexer::SkipEscapedNewLines(const char *P) { } } +const char *Lexer::SkipHorizontalWhitespace(const char *Ptr) { + // Small amounts of horizontal whitespace is very common between tokens. + // Check for space character separately to skip the expensive + // isHorizontalWhitespace() check + if (*Ptr == ' ' || isHorizontalWhitespace(*Ptr)) { + do { + ++Ptr; + } while (*Ptr == ' ' || isHorizontalWhitespace(*Ptr)); + } + return Ptr; +} + std::optional<Token> Lexer::findNextToken(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts, @@ -3833,16 +3845,12 @@ bool Lexer::LexTokenInternal(Token &Result) { assert(!Result.hasPtrData() && "Result has not been reset"); // CurPtr - Cache BufferPtr in an automatic variable. - const char *CurPtr = BufferPtr; - - // Small amounts of horizontal whitespace is very common between tokens. - // Check for space character separately to skip the expensive - // isHorizontalWhitespace() check - if (*CurPtr == ' ' || isHorizontalWhitespace(*CurPtr)) { - do { - ++CurPtr; - } while (*CurPtr == ' ' || isHorizontalWhitespace(*CurPtr)); + const char *CurPtr = SkipHorizontalWhitespace(BufferPtr); + /// CurPtr has been advanced forward, indicating that a horizontal whitespace + /// character has been encountered. Check if the Lexer is in keep whitespace + /// mode. + if (CurPtr != BufferPtr) { // If we are keeping whitespace and other tokens, just return what we just // skipped. The next lexer invocation will return the token after the // whitespace. diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp index 427a3826b299a..d4a6559f3a504 100644 --- a/clang/lib/Lex/Preprocessor.cpp +++ b/clang/lib/Lex/Preprocessor.cpp @@ -1356,11 +1356,33 @@ bool Preprocessor::HandleModuleContextualKeyword(Token &Result) { llvm::SaveAndRestore<bool> SavedParsingPreprocessorDirective( CurPPLexer->ParsingPreprocessorDirective, true); - // The next token may be an angled string literal after import keyword. - llvm::SaveAndRestore<bool> SavedParsingFilemame( - CurPPLexer->ParsingFilename, - Result.getIdentifierInfo()->isImportKeyword()); + bool ParsingFilename = false; + if (Result.getIdentifierInfo()->isImportKeyword()) { + if (getLangOpts().Digraphs && CurLexer && + CurLexer->getCurrentBufferOffset() + 2 < CurLexer->getBuffer().size()) { + // If the import preprocessing token folled by a digraph character '<:', + // the import preprocessing should not traited as a import contextual + // keyword. Eg. + // int + // import <:10 + // :>; + // + // This is a array definition, and equivalent to: + // + // int import[10]; + const char *CurPtr = CurLexer->getBufferLocation(); + CurPtr = Lexer::SkipHorizontalWhitespace(CurPtr); + auto C0 = Lexer::getCharAndSizeNoWarn(CurPtr, getLangOpts()); + auto C1 = Lexer::getCharAndSizeNoWarn(CurPtr + C0.Size, getLangOpts()); + if (C0.Char == '<' && (C1.Char == ':' || C1.Char == '%')) + return false; + } + ParsingFilename = true; + } + // The next token may be an angled string literal after import keyword. + llvm::SaveAndRestore<bool> SavedParsingFilemame(CurPPLexer->ParsingFilename, + ParsingFilename); std::optional<Token> NextTok = peekNextPPToken(); if (!NextTok) return false; diff --git a/clang/test/CXX/module/cpp.pre/p1.cpp b/clang/test/CXX/module/cpp.pre/p1.cpp index 989915004ff57..0e2fb65390e99 100644 --- a/clang/test/CXX/module/cpp.pre/p1.cpp +++ b/clang/test/CXX/module/cpp.pre/p1.cpp @@ -38,6 +38,8 @@ // RUN: %clang_cc1 -std=c++20 %t/func_like_macro.cpp -D'm(x)=x' -fsyntax-only -verify // RUN: %clang_cc1 -std=c++20 %t/lparen.cpp -D'm(x)=x' -D'LPAREN=(' -fsyntax-only -verify // RUN: %clang_cc1 -std=c++20 %t/control_line.cpp -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 %t/digraph.cpp -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 %t/digraph2.cpp -fsyntax-only -verify //--- hash.cpp @@ -205,3 +207,27 @@ export module m; // expected-error {{module directive lines are not allowed on l // expected-error {{module declaration must occur at the start of the translation unit}} \ // expected-note@#1 {{add 'module;'}} #endif + +//--- digraph.cpp +// expected-no-diagnostics +int +import <:10 +:>; + +void foo() { + for (int i = 0; i < 10; ++i) + import[i] = i; +} + +//--- digraph2.cpp +// expected-no-diagnostics +using import = int; + +void bar(int); + +void foo(int val = +import <%%> +) { + bar(val); +} + >From 4662a1555e4da42b375a4fe5df3547efebef346e Mon Sep 17 00:00:00 2001 From: yronglin <[email protected]> Date: Thu, 9 Apr 2026 21:54:18 +0800 Subject: [PATCH 02/14] Revert "[C++][Modules][Preprocessor] Clang should not convert a import preprocessing token to contextual keyword if a digraph character following import" This reverts commit 373880aed203efd8521dfb76a3f52fedee2592dc. --- clang/include/clang/Lex/Lexer.h | 4 ---- clang/lib/Lex/Lexer.cpp | 26 ++++++++-------------- clang/lib/Lex/Preprocessor.cpp | 33 +++++----------------------- clang/test/CXX/module/cpp.pre/p1.cpp | 26 ---------------------- 4 files changed, 15 insertions(+), 74 deletions(-) diff --git a/clang/include/clang/Lex/Lexer.h b/clang/include/clang/Lex/Lexer.h index cd872cfcebb7e..b042e5fb088fa 100644 --- a/clang/include/clang/Lex/Lexer.h +++ b/clang/include/clang/Lex/Lexer.h @@ -740,10 +740,6 @@ class Lexer : public PreprocessorLexer { /// otherwise return P. static const char *SkipEscapedNewLines(const char *P); - /// SkipHorizontalWhitespace - Skip the horizontak whitespace characters and - /// returns the advanced pointer. - static const char *SkipHorizontalWhitespace(const char *Ptr); - /// getCharAndSizeSlowNoWarn - Same as getCharAndSizeSlow, but never emits a /// diagnostic. static SizedChar getCharAndSizeSlowNoWarn(const char *Ptr, diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp index c5e0a8314d185..f7d44879a936d 100644 --- a/clang/lib/Lex/Lexer.cpp +++ b/clang/lib/Lex/Lexer.cpp @@ -1372,18 +1372,6 @@ const char *Lexer::SkipEscapedNewLines(const char *P) { } } -const char *Lexer::SkipHorizontalWhitespace(const char *Ptr) { - // Small amounts of horizontal whitespace is very common between tokens. - // Check for space character separately to skip the expensive - // isHorizontalWhitespace() check - if (*Ptr == ' ' || isHorizontalWhitespace(*Ptr)) { - do { - ++Ptr; - } while (*Ptr == ' ' || isHorizontalWhitespace(*Ptr)); - } - return Ptr; -} - std::optional<Token> Lexer::findNextToken(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts, @@ -3845,12 +3833,16 @@ bool Lexer::LexTokenInternal(Token &Result) { assert(!Result.hasPtrData() && "Result has not been reset"); // CurPtr - Cache BufferPtr in an automatic variable. - const char *CurPtr = SkipHorizontalWhitespace(BufferPtr); + const char *CurPtr = BufferPtr; + + // Small amounts of horizontal whitespace is very common between tokens. + // Check for space character separately to skip the expensive + // isHorizontalWhitespace() check + if (*CurPtr == ' ' || isHorizontalWhitespace(*CurPtr)) { + do { + ++CurPtr; + } while (*CurPtr == ' ' || isHorizontalWhitespace(*CurPtr)); - /// CurPtr has been advanced forward, indicating that a horizontal whitespace - /// character has been encountered. Check if the Lexer is in keep whitespace - /// mode. - if (CurPtr != BufferPtr) { // If we are keeping whitespace and other tokens, just return what we just // skipped. The next lexer invocation will return the token after the // whitespace. diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp index d4a6559f3a504..7f628aeb73a0a 100644 --- a/clang/lib/Lex/Preprocessor.cpp +++ b/clang/lib/Lex/Preprocessor.cpp @@ -1356,34 +1356,13 @@ bool Preprocessor::HandleModuleContextualKeyword(Token &Result) { llvm::SaveAndRestore<bool> SavedParsingPreprocessorDirective( CurPPLexer->ParsingPreprocessorDirective, true); - bool ParsingFilename = false; - if (Result.getIdentifierInfo()->isImportKeyword()) { - if (getLangOpts().Digraphs && CurLexer && - CurLexer->getCurrentBufferOffset() + 2 < CurLexer->getBuffer().size()) { - // If the import preprocessing token folled by a digraph character '<:', - // the import preprocessing should not traited as a import contextual - // keyword. Eg. - // int - // import <:10 - // :>; - // - // This is a array definition, and equivalent to: - // - // int import[10]; - const char *CurPtr = CurLexer->getBufferLocation(); - CurPtr = Lexer::SkipHorizontalWhitespace(CurPtr); - auto C0 = Lexer::getCharAndSizeNoWarn(CurPtr, getLangOpts()); - auto C1 = Lexer::getCharAndSizeNoWarn(CurPtr + C0.Size, getLangOpts()); - if (C0.Char == '<' && (C1.Char == ':' || C1.Char == '%')) - return false; - } - ParsingFilename = true; - } - // The next token may be an angled string literal after import keyword. - llvm::SaveAndRestore<bool> SavedParsingFilemame(CurPPLexer->ParsingFilename, - ParsingFilename); - std::optional<Token> NextTok = peekNextPPToken(); + llvm::SaveAndRestore<bool> SavedParsingFilemame( + CurPPLexer->ParsingFilename, + Result.getIdentifierInfo()->isImportKeyword()); + + std::optional<Token> NextTok = + CurLexer ? CurLexer->peekNextPPToken() : CurTokenLexer->peekNextPPToken(); if (!NextTok) return false; diff --git a/clang/test/CXX/module/cpp.pre/p1.cpp b/clang/test/CXX/module/cpp.pre/p1.cpp index 0e2fb65390e99..989915004ff57 100644 --- a/clang/test/CXX/module/cpp.pre/p1.cpp +++ b/clang/test/CXX/module/cpp.pre/p1.cpp @@ -38,8 +38,6 @@ // RUN: %clang_cc1 -std=c++20 %t/func_like_macro.cpp -D'm(x)=x' -fsyntax-only -verify // RUN: %clang_cc1 -std=c++20 %t/lparen.cpp -D'm(x)=x' -D'LPAREN=(' -fsyntax-only -verify // RUN: %clang_cc1 -std=c++20 %t/control_line.cpp -fsyntax-only -verify -// RUN: %clang_cc1 -std=c++20 %t/digraph.cpp -fsyntax-only -verify -// RUN: %clang_cc1 -std=c++20 %t/digraph2.cpp -fsyntax-only -verify //--- hash.cpp @@ -207,27 +205,3 @@ export module m; // expected-error {{module directive lines are not allowed on l // expected-error {{module declaration must occur at the start of the translation unit}} \ // expected-note@#1 {{add 'module;'}} #endif - -//--- digraph.cpp -// expected-no-diagnostics -int -import <:10 -:>; - -void foo() { - for (int i = 0; i < 10; ++i) - import[i] = i; -} - -//--- digraph2.cpp -// expected-no-diagnostics -using import = int; - -void bar(int); - -void foo(int val = -import <%%> -) { - bar(val); -} - >From 593b498dfc932b9de5acabf751c5ef3316dc84e7 Mon Sep 17 00:00:00 2001 From: yronglin <[email protected]> Date: Thu, 9 Apr 2026 23:53:22 +0800 Subject: [PATCH 03/14] [C++][Modules] Don't check '<' after 'import' when converting import pp-token to contextual keyword Signed-off-by: yronglin <[email protected]> --- clang/lib/Lex/Preprocessor.cpp | 2 +- clang/lib/Parse/Parser.cpp | 10 ++++++++ clang/test/CXX/module/cpp.pre/p1.cpp | 36 +++++++++++++++++++++++++++- 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp index 7f628aeb73a0a..2c998497d72f7 100644 --- a/clang/lib/Lex/Preprocessor.cpp +++ b/clang/lib/Lex/Preprocessor.cpp @@ -1370,7 +1370,7 @@ bool Preprocessor::HandleModuleContextualKeyword(Token &Result) { LookUpIdentifierInfo(*NextTok); if (Result.getIdentifierInfo()->isImportKeyword()) { - if (NextTok->isOneOf(tok::identifier, tok::less, tok::colon, + if (NextTok->isOneOf(tok::identifier, tok::colon, tok::header_name)) { Result.setKind(tok::kw_import); ModuleImportLoc = Result.getLocation(); diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp index 6a21acd9dc4ef..d83b75072f844 100644 --- a/clang/lib/Parse/Parser.cpp +++ b/clang/lib/Parse/Parser.cpp @@ -2507,6 +2507,16 @@ Decl *Parser::ParseModuleImport(SourceLocation AtLoc, break; } + // FIXME: If the previous token is tok::header_name like the following: + // + // import <%%> + // + // The diagnostic location is incorrect. + // + // <source file>:1:10: error: import directive must end with a ';' + // 1 | import <%%> + // | ^ + // | ; bool LexedSemi = false; if (getLangOpts().CPlusPlusModules) LexedSemi = diff --git a/clang/test/CXX/module/cpp.pre/p1.cpp b/clang/test/CXX/module/cpp.pre/p1.cpp index 989915004ff57..d0cf0ee8efe1a 100644 --- a/clang/test/CXX/module/cpp.pre/p1.cpp +++ b/clang/test/CXX/module/cpp.pre/p1.cpp @@ -38,7 +38,10 @@ // RUN: %clang_cc1 -std=c++20 %t/func_like_macro.cpp -D'm(x)=x' -fsyntax-only -verify // RUN: %clang_cc1 -std=c++20 %t/lparen.cpp -D'm(x)=x' -D'LPAREN=(' -fsyntax-only -verify // RUN: %clang_cc1 -std=c++20 %t/control_line.cpp -fsyntax-only -verify - +// RUN: %clang_cc1 -std=c++20 %t/digraph.cpp -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 %t/digraph2.cpp -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 %t/digraph3.cpp -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 %t/digraph4.cpp -fsyntax-only -verify //--- hash.cpp // expected-no-diagnostics @@ -205,3 +208,34 @@ export module m; // expected-error {{module directive lines are not allowed on l // expected-error {{module declaration must occur at the start of the translation unit}} \ // expected-note@#1 {{add 'module;'}} #endif + +//--- digraph.cpp +// expected-no-diagnostics +int +import <:10 +:>; + +void foo() { + for (int i = 0; i < 10; ++i) + import[i] = i; +} + +//--- digraph2.cpp +// expected-no-diagnostics +using import = int; + +void bar(int); + +void foo(int val = +import <% +%> +) { + bar(val); +} + +//--- digraph3.cpp +import <%%>; // expected-error {{'%%' file not found}} + +//--- digraph4.cpp +import <::>; // expected-error {{'::' file not found}} + >From 20a94b95ecac89a9983b0eaffe4fc5ba33dd5c66 Mon Sep 17 00:00:00 2001 From: yronglin <[email protected]> Date: Sun, 12 Apr 2026 02:45:48 +0800 Subject: [PATCH 04/14] [Clang][Preprocessor] Unify header-name lookahead for import and include Introduce Preprocessor::isNextPPTokenHeaderNameOrOneOf to centralize lookahead logic for header-name formation and token classification under ParsingFilename mode. Refactor handling of C++20 module/import contextual keywords and LexHeaderName to use the new helper, ensuring consistent behavior between `import` and `#include`. This fixes incorrect acceptance of cases where macro expansion after a digraph-like `<:` leads to invalid header-name parsing, e.g.: #define FOO foo> #include <:FOO Now such cases are rejected as expected. Also adjusts peekNextPPToken to properly support dependency directive lexers. No functional change intended for valid code; improves correctness and consistency in edge cases involving header-name lexing. Signed-off-by: yronglin <[email protected]> --- clang/include/clang/Lex/Preprocessor.h | 23 +++++++++++++- clang/lib/Lex/Lexer.cpp | 18 +++++------ clang/lib/Lex/Preprocessor.cpp | 43 ++++++++++++-------------- clang/test/CXX/cpp/cpp.include/p3.cpp | 5 +++ clang/test/CXX/module/cpp.pre/p1.cpp | 33 ++++++++++++++------ 5 files changed, 78 insertions(+), 44 deletions(-) create mode 100644 clang/test/CXX/cpp/cpp.include/p3.cpp diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h index 1c917dcfe7b7e..2b97b645955cc 100644 --- a/clang/include/clang/Lex/Preprocessor.h +++ b/clang/include/clang/Lex/Preprocessor.h @@ -49,6 +49,7 @@ #include "llvm/Support/Allocator.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Registry.h" +#include "llvm/Support/SaveAndRestore.h" #include "llvm/Support/TrailingObjects.h" #include <cassert> #include <cstddef> @@ -1827,6 +1828,26 @@ class Preprocessor { void HandleCXXImportDirective(Token Import); void HandleCXXModuleDirective(Token Module); + template <typename... Ts> bool isNextPPTokenHeaderNameOrOneOf(Ts... Ks) { + // First, tries to form a valid header-name token. + llvm::SaveAndRestore<bool> SavedFilename(CurPPLexer->ParsingFilename, + true); + if (auto Tok = peekNextPPToken()) { + if (Tok->is(tok::header_name)) + return true; + } + + // If that fails and it's not one of the other tokens, then it's not a + // directive. + CurPPLexer->ParsingFilename = false; + if (auto NextTok = peekNextPPToken()) { + if (NextTok->is(tok::raw_identifier)) + LookUpIdentifierInfo(*NextTok); + return NextTok->isOneOf(Ks...); + } + return false; + } + /// Callback invoked when the lexer sees one of export, import or module token /// at the start of a line. /// @@ -2377,12 +2398,12 @@ class Preprocessor { return NextTokOpt.has_value() ? NextTokOpt->is(Ks...) : false; } -private: /// peekNextPPToken - Return std::nullopt if there are no more tokens in the /// buffer controlled by this lexer, otherwise return the next unexpanded /// token. std::optional<Token> peekNextPPToken() const; +private: /// Identifiers used for SEH handling in Borland. These are only /// allowed in particular circumstances // __except block diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp index f7d44879a936d..f05788e669223 100644 --- a/clang/lib/Lex/Lexer.cpp +++ b/clang/lib/Lex/Lexer.cpp @@ -3287,15 +3287,6 @@ bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) { std::optional<Token> Lexer::peekNextPPToken() { assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?"); - if (isDependencyDirectivesLexer()) { - if (NextDepDirectiveTokenIndex == DepDirectives.front().Tokens.size()) - return std::nullopt; - Token Result; - (void)convertDependencyDirectiveToken( - DepDirectives.front().Tokens[NextDepDirectiveTokenIndex], Result); - return Result; - } - // Switch to 'skipping' mode. This will ensure that we can lex a token // without emitting diagnostics, disables macro expansion, and will cause EOF // to return an EOF token instead of popping the include stack. @@ -3310,7 +3301,14 @@ std::optional<Token> Lexer::peekNextPPToken() { MultipleIncludeOpt MIOptState = MIOpt; Token Tok; - Lex(Tok); + if (isDependencyDirectivesLexer()) { + if (NextDepDirectiveTokenIndex == DepDirectives.front().Tokens.size()) + return std::nullopt; + (void)convertDependencyDirectiveToken( + DepDirectives.front().Tokens[NextDepDirectiveTokenIndex], Tok); + } else { + Lex(Tok); + } // Restore state that may have changed. BufferPtr = TmpBufferPtr; diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp index 2c998497d72f7..2490742bcea8f 100644 --- a/clang/lib/Lex/Preprocessor.cpp +++ b/clang/lib/Lex/Preprocessor.cpp @@ -1074,8 +1074,11 @@ bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) { // __has_include(__has_include)) if (CurPPLexer->ParsingFilename) LexUnexpandedToken(FilenameTok); - else + else if ((getLangOpts().CPlusPlusModules && isImportingCXXNamedModules()) || + isNextPPTokenHeaderNameOrOneOf(tok::less)) CurPPLexer->LexIncludeFilename(FilenameTok); + else + Lex(FilenameTok); } else { Lex(FilenameTok); } @@ -1356,32 +1359,24 @@ bool Preprocessor::HandleModuleContextualKeyword(Token &Result) { llvm::SaveAndRestore<bool> SavedParsingPreprocessorDirective( CurPPLexer->ParsingPreprocessorDirective, true); - // The next token may be an angled string literal after import keyword. - llvm::SaveAndRestore<bool> SavedParsingFilemame( - CurPPLexer->ParsingFilename, - Result.getIdentifierInfo()->isImportKeyword()); - - std::optional<Token> NextTok = - CurLexer ? CurLexer->peekNextPPToken() : CurTokenLexer->peekNextPPToken(); - if (!NextTok) - return false; - - if (NextTok->is(tok::raw_identifier)) - LookUpIdentifierInfo(*NextTok); - - if (Result.getIdentifierInfo()->isImportKeyword()) { - if (NextTok->isOneOf(tok::identifier, tok::colon, - tok::header_name)) { - Result.setKind(tok::kw_import); - ModuleImportLoc = Result.getLocation(); - return true; + if (II->isModuleKeyword()) { + if (auto NextTok = peekNextPPToken()) { + if (NextTok->is(tok::raw_identifier)) + LookUpIdentifierInfo(*NextTok); + if (NextTok->isOneOf(tok::identifier, tok::colon, tok::semi)) { + Result.setKind(tok::kw_module); + ModuleDeclLoc = Result.getLocation(); + return true; + } } + return false; } - if (Result.getIdentifierInfo()->isModuleKeyword() && - NextTok->isOneOf(tok::identifier, tok::colon, tok::semi)) { - Result.setKind(tok::kw_module); - ModuleDeclLoc = Result.getLocation(); + if (II->isImportKeyword() && + isNextPPTokenHeaderNameOrOneOf(tok::identifier, tok::colon, tok::less)) { + Result.setKind(tok::kw_import); + ModuleImportLoc = Result.getLocation(); + IsAtImport = false; return true; } diff --git a/clang/test/CXX/cpp/cpp.include/p3.cpp b/clang/test/CXX/cpp/cpp.include/p3.cpp new file mode 100644 index 0000000000000..7afb4af1c9423 --- /dev/null +++ b/clang/test/CXX/cpp/cpp.include/p3.cpp @@ -0,0 +1,5 @@ +// RUN: %clang_cc1 %s -fsyntax-only -verify + +#define FOO foo> +#include <:FOO +// expected-error@-1 {{expected "FILENAME" or <FILENAME>}} diff --git a/clang/test/CXX/module/cpp.pre/p1.cpp b/clang/test/CXX/module/cpp.pre/p1.cpp index d0cf0ee8efe1a..5b6f225f2f58c 100644 --- a/clang/test/CXX/module/cpp.pre/p1.cpp +++ b/clang/test/CXX/module/cpp.pre/p1.cpp @@ -38,11 +38,12 @@ // RUN: %clang_cc1 -std=c++20 %t/func_like_macro.cpp -D'm(x)=x' -fsyntax-only -verify // RUN: %clang_cc1 -std=c++20 %t/lparen.cpp -D'm(x)=x' -D'LPAREN=(' -fsyntax-only -verify // RUN: %clang_cc1 -std=c++20 %t/control_line.cpp -fsyntax-only -verify -// RUN: %clang_cc1 -std=c++20 %t/digraph.cpp -fsyntax-only -verify -// RUN: %clang_cc1 -std=c++20 %t/digraph2.cpp -fsyntax-only -verify -// RUN: %clang_cc1 -std=c++20 %t/digraph3.cpp -fsyntax-only -verify -// RUN: %clang_cc1 -std=c++20 %t/digraph4.cpp -fsyntax-only -verify - +// RUN: %clang_cc1 -std=c++20 %t/header_name1.cpp -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 %t/header_name2.cpp -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 %t/header_name3.cpp -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 %t/header_name4.cpp -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 %t/header_name5.cpp -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 %t/header_name6.cpp -fsyntax-only -verify //--- hash.cpp // expected-no-diagnostics # // preprocessing directive @@ -209,7 +210,7 @@ export module m; // expected-error {{module directive lines are not allowed on l // expected-note@#1 {{add 'module;'}} #endif -//--- digraph.cpp +//--- header_name1.cpp // expected-no-diagnostics int import <:10 @@ -220,7 +221,7 @@ void foo() { import[i] = i; } -//--- digraph2.cpp +//--- header_name2.cpp // expected-no-diagnostics using import = int; @@ -233,9 +234,23 @@ import <% bar(val); } -//--- digraph3.cpp +//--- header_name3.cpp +export module M; import <%%>; // expected-error {{'%%' file not found}} -//--- digraph4.cpp +//--- header_name4.cpp +export module M; import <::>; // expected-error {{'::' file not found}} +//--- header_name5.cpp +export module M; +#define FOO foo>; +import <:FOO +// expected-error@-1 {{use of undeclared identifier 'foo'}} +// expected-error@-2 {{a type specifier is required for all declarations}} +// expected-error@-3 {{expected expression}} + +//--- header_name6.cpp +export module M; +#define HEADER vector> +import <HEADER; // expected-error {{file not found}} >From 03ae144f0a2a807c57f478dcc6358b88bbaa3357 Mon Sep 17 00:00:00 2001 From: yronglin <[email protected]> Date: Sun, 12 Apr 2026 02:56:10 +0800 Subject: [PATCH 05/14] Format Signed-off-by: yronglin <[email protected]> --- clang/include/clang/Lex/Preprocessor.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h index 2b97b645955cc..4c9ef006fd7ad 100644 --- a/clang/include/clang/Lex/Preprocessor.h +++ b/clang/include/clang/Lex/Preprocessor.h @@ -1830,8 +1830,8 @@ class Preprocessor { template <typename... Ts> bool isNextPPTokenHeaderNameOrOneOf(Ts... Ks) { // First, tries to form a valid header-name token. - llvm::SaveAndRestore<bool> SavedFilename(CurPPLexer->ParsingFilename, - true); + llvm::SaveAndRestore<bool> SavedParsingFilename(CurPPLexer->ParsingFilename, + true); if (auto Tok = peekNextPPToken()) { if (Tok->is(tok::header_name)) return true; >From c94381c83a1b137bbfa9ee46bfe975136207bb90 Mon Sep 17 00:00:00 2001 From: yronglin <[email protected]> Date: Mon, 13 Apr 2026 01:09:45 +0800 Subject: [PATCH 06/14] Avoid 2nd peekNextPPToken call Signed-off-by: yronglin <[email protected]> --- clang/include/clang/Lex/Preprocessor.h | 75 ++++++++++++++++++-------- 1 file changed, 54 insertions(+), 21 deletions(-) diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h index 4c9ef006fd7ad..32d0c8b8ede1e 100644 --- a/clang/include/clang/Lex/Preprocessor.h +++ b/clang/include/clang/Lex/Preprocessor.h @@ -1828,26 +1828,6 @@ class Preprocessor { void HandleCXXImportDirective(Token Import); void HandleCXXModuleDirective(Token Module); - template <typename... Ts> bool isNextPPTokenHeaderNameOrOneOf(Ts... Ks) { - // First, tries to form a valid header-name token. - llvm::SaveAndRestore<bool> SavedParsingFilename(CurPPLexer->ParsingFilename, - true); - if (auto Tok = peekNextPPToken()) { - if (Tok->is(tok::header_name)) - return true; - } - - // If that fails and it's not one of the other tokens, then it's not a - // directive. - CurPPLexer->ParsingFilename = false; - if (auto NextTok = peekNextPPToken()) { - if (NextTok->is(tok::raw_identifier)) - LookUpIdentifierInfo(*NextTok); - return NextTok->isOneOf(Ks...); - } - return false; - } - /// Callback invoked when the lexer sees one of export, import or module token /// at the start of a line. /// @@ -2398,12 +2378,65 @@ class Preprocessor { return NextTokOpt.has_value() ? NextTokOpt->is(Ks...) : false; } +private: /// peekNextPPToken - Return std::nullopt if there are no more tokens in the /// buffer controlled by this lexer, otherwise return the next unexpanded /// token. std::optional<Token> peekNextPPToken() const; -private: + /// Check whether the next preprocessing token can form a header-name token + /// or matches one of the specified token kinds. + /// + /// This performs a lookahead without consuming tokens: + /// - First, it temporarily enables `ParsingFilename` to attempt forming a + /// `tok::header_name` (e.g. `<foo>` or "foo"). + /// - If that succeeds, returns true. + /// - Otherwise, it restores normal lexing mode and checks whether the next + /// token matches any of the provided kinds `Ks...`. + /// + /// This helper is used to classify tokens in contexts such as C++20 `import` + /// and `#include`, ensuring consistent handling of header-name lexing and + /// avoiding unintended lexer state changes. + template <typename... Ts> bool isNextPPTokenHeaderNameOrOneOf(Ts... Ks) { + // First, tries to form a valid header-name token. + llvm::SaveAndRestore<bool> SavedParsingFilename(CurPPLexer->ParsingFilename, + true); + if (auto NextTok = peekNextPPToken()) { + if (NextTok->is(tok::header_name)) + return true; + + // In ParsingFilename mode, both <...> and "..." are lexed as header-name + // tokens. If a valid header-name is formed, return immediately. + // + // Otherwise, we may need to re-lex the token in normal mode. This is + // required for '<' to correctly handle cases such as digraphs ('<:', + // '<%') and situations where macro expansion affects token boundaries, + // e.g.: + // + // #define VECTOR vector> + // #include <VECTOR + // + // In such cases, the initial lex in ParsingFilename mode may fail to form + // a header-name, and only normal lexing yields the correct tokenization. + // For all other tokens, the result is identical between the two modes, so + // we can classify them directly and avoid calling the relatively + // expensive second peekNextPPToken() on the common path. + if (NextTok->isNot(tok::less)) { + if (NextTok->is(tok::raw_identifier)) + LookUpIdentifierInfo(*NextTok); + return NextTok->isOneOf(Ks...); + } + } + + // If that fails and it's not one of the other tokens, then it's not a + // directive. + CurPPLexer->ParsingFilename = false; + if (auto NextTok = peekNextPPToken()) { + return NextTok->isOneOf(Ks...); + } + return false; + } + /// Identifiers used for SEH handling in Borland. These are only /// allowed in particular circumstances // __except block >From c1f40b4cebb3411f46d1fd0651d8dc524032bb45 Mon Sep 17 00:00:00 2001 From: yronglin <[email protected]> Date: Mon, 13 Apr 2026 01:20:04 +0800 Subject: [PATCH 07/14] Add comments in LexHeaderName Signed-off-by: yronglin <[email protected]> --- clang/lib/Lex/Preprocessor.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp index 2490742bcea8f..1515d0dc09b9d 100644 --- a/clang/lib/Lex/Preprocessor.cpp +++ b/clang/lib/Lex/Preprocessor.cpp @@ -1074,7 +1074,11 @@ bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) { // __has_include(__has_include)) if (CurPPLexer->ParsingFilename) LexUnexpandedToken(FilenameTok); - else if ((getLangOpts().CPlusPlusModules && isImportingCXXNamedModules()) || + else if ((getLangOpts().CPlusPlusModules && + isImportingCXXNamedModules()) || // C++ import already checked in + // HandleModuleContextualKeyword, + // avoid duplicate check in + // LexHeaderName. isNextPPTokenHeaderNameOrOneOf(tok::less)) CurPPLexer->LexIncludeFilename(FilenameTok); else >From 0215f2016803fa45e8ab01e39e146d88d9ecc44e Mon Sep 17 00:00:00 2001 From: yronglin <[email protected]> Date: Thu, 16 Apr 2026 14:33:03 +0800 Subject: [PATCH 08/14] Fix rebase issue --- clang/lib/Lex/Preprocessor.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp index 1515d0dc09b9d..1c8855b3365ba 100644 --- a/clang/lib/Lex/Preprocessor.cpp +++ b/clang/lib/Lex/Preprocessor.cpp @@ -1380,7 +1380,6 @@ bool Preprocessor::HandleModuleContextualKeyword(Token &Result) { isNextPPTokenHeaderNameOrOneOf(tok::identifier, tok::colon, tok::less)) { Result.setKind(tok::kw_import); ModuleImportLoc = Result.getLocation(); - IsAtImport = false; return true; } >From fb7822bb11062111c99e759d0c11c36fdc57bbe3 Mon Sep 17 00:00:00 2001 From: Yihan Wang <[email protected]> Date: Sun, 26 Jul 2026 02:05:54 +0800 Subject: [PATCH 09/14] [clang] If lexing an angled header name fails, it's simply wrong to assume that we should produce a < token. We still need to check for the other kinds of token that start with a <. Signed-off-by: Yihan Wang <[email protected]> --- clang/include/clang/Lex/Preprocessor.h | 53 -------------------------- clang/lib/Lex/Lexer.cpp | 20 ++++++---- clang/lib/Lex/Preprocessor.cpp | 27 ++++++------- 3 files changed, 26 insertions(+), 74 deletions(-) diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h index 32d0c8b8ede1e..d3633fef9114d 100644 --- a/clang/include/clang/Lex/Preprocessor.h +++ b/clang/include/clang/Lex/Preprocessor.h @@ -2384,59 +2384,6 @@ class Preprocessor { /// token. std::optional<Token> peekNextPPToken() const; - /// Check whether the next preprocessing token can form a header-name token - /// or matches one of the specified token kinds. - /// - /// This performs a lookahead without consuming tokens: - /// - First, it temporarily enables `ParsingFilename` to attempt forming a - /// `tok::header_name` (e.g. `<foo>` or "foo"). - /// - If that succeeds, returns true. - /// - Otherwise, it restores normal lexing mode and checks whether the next - /// token matches any of the provided kinds `Ks...`. - /// - /// This helper is used to classify tokens in contexts such as C++20 `import` - /// and `#include`, ensuring consistent handling of header-name lexing and - /// avoiding unintended lexer state changes. - template <typename... Ts> bool isNextPPTokenHeaderNameOrOneOf(Ts... Ks) { - // First, tries to form a valid header-name token. - llvm::SaveAndRestore<bool> SavedParsingFilename(CurPPLexer->ParsingFilename, - true); - if (auto NextTok = peekNextPPToken()) { - if (NextTok->is(tok::header_name)) - return true; - - // In ParsingFilename mode, both <...> and "..." are lexed as header-name - // tokens. If a valid header-name is formed, return immediately. - // - // Otherwise, we may need to re-lex the token in normal mode. This is - // required for '<' to correctly handle cases such as digraphs ('<:', - // '<%') and situations where macro expansion affects token boundaries, - // e.g.: - // - // #define VECTOR vector> - // #include <VECTOR - // - // In such cases, the initial lex in ParsingFilename mode may fail to form - // a header-name, and only normal lexing yields the correct tokenization. - // For all other tokens, the result is identical between the two modes, so - // we can classify them directly and avoid calling the relatively - // expensive second peekNextPPToken() on the common path. - if (NextTok->isNot(tok::less)) { - if (NextTok->is(tok::raw_identifier)) - LookUpIdentifierInfo(*NextTok); - return NextTok->isOneOf(Ks...); - } - } - - // If that fails and it's not one of the other tokens, then it's not a - // directive. - CurPPLexer->ParsingFilename = false; - if (auto NextTok = peekNextPPToken()) { - return NextTok->isOneOf(Ks...); - } - return false; - } - /// Identifiers used for SEH handling in Borland. These are only /// allowed in particular circumstances // __except block diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp index f05788e669223..2badd1488daee 100644 --- a/clang/lib/Lex/Lexer.cpp +++ b/clang/lib/Lex/Lexer.cpp @@ -2446,6 +2446,8 @@ bool Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr, /// LexAngledStringLiteral - Lex the remainder of an angled string literal, /// after having lexed the '<' character. This is used for #include filenames. +/// Returns false if failed to lex the angled string literal; so the caller can +/// lex the '<' normally. bool Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) { // Does this string contain the \0 character? const char *NulCharacter = nullptr; @@ -2459,10 +2461,8 @@ bool Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) { if (isVerticalWhitespace(C) || // Newline. (C == 0 && (CurPtr - 1 == BufferEnd))) { // End of file. - // If the filename is unterminated, then it must just be a lone < - // character. Return this as such. - FormTokenWithChars(Result, AfterLessPos, tok::less); - return true; + // If the filename is unterminated, let the caller lex the '<' normally. + return false; } if (C == 0) { @@ -4326,9 +4326,10 @@ bool Lexer::LexTokenInternal(Token &Result) { break; case '<': Char = getCharAndSize(CurPtr, SizeTmp); - if (ParsingFilename) { - return LexAngledStringLiteral(Result, CurPtr); - } else if (Char == '<') { + if (ParsingFilename && LexAngledStringLiteral(Result, CurPtr)) + return true; + + if (Char == '<') { char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); if (After == '=') { Kind = tok::lesslessequal; @@ -4675,7 +4676,10 @@ bool Lexer::LexDependencyDirectiveToken(Token &Result) { if (ParsingFilename && DDTok.is(tok::less)) { BufferPtr = BufferStart + DDTok.Offset; - LexAngledStringLiteral(Result, BufferPtr + 1); + if (!LexAngledStringLiteral(Result, BufferPtr + 1)) { + convertDependencyDirectiveToken(DDTok, Result); + return true; + } if (Result.isNot(tok::header_name)) return true; // Advance the index of lexed tokens. diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp index 1c8855b3365ba..ea9064cc8a465 100644 --- a/clang/lib/Lex/Preprocessor.cpp +++ b/clang/lib/Lex/Preprocessor.cpp @@ -1074,15 +1074,8 @@ bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) { // __has_include(__has_include)) if (CurPPLexer->ParsingFilename) LexUnexpandedToken(FilenameTok); - else if ((getLangOpts().CPlusPlusModules && - isImportingCXXNamedModules()) || // C++ import already checked in - // HandleModuleContextualKeyword, - // avoid duplicate check in - // LexHeaderName. - isNextPPTokenHeaderNameOrOneOf(tok::less)) - CurPPLexer->LexIncludeFilename(FilenameTok); else - Lex(FilenameTok); + CurPPLexer->LexIncludeFilename(FilenameTok); } else { Lex(FilenameTok); } @@ -1376,11 +1369,19 @@ bool Preprocessor::HandleModuleContextualKeyword(Token &Result) { return false; } - if (II->isImportKeyword() && - isNextPPTokenHeaderNameOrOneOf(tok::identifier, tok::colon, tok::less)) { - Result.setKind(tok::kw_import); - ModuleImportLoc = Result.getLocation(); - return true; + if (II->isImportKeyword()) { + llvm::SaveAndRestore<bool> SavedParsingFilename(CurPPLexer->ParsingFilename, + true); + if (auto NextTok = peekNextPPToken()) { + if (NextTok->is(tok::raw_identifier)) + LookUpIdentifierInfo(*NextTok); + if (NextTok->isOneOf(tok::header_name, tok::identifier, tok::colon, tok::less)) { + Result.setKind(tok::kw_import); + ModuleImportLoc = Result.getLocation(); + return true; + } + } + return false; } // Ok, it's an identifier. >From f497e975289249a62cafd575c5cf95bc809b7849 Mon Sep 17 00:00:00 2001 From: Yihan Wang <[email protected]> Date: Sun, 26 Jul 2026 02:10:04 +0800 Subject: [PATCH 10/14] Remove unnecessary include Signed-off-by: Yihan Wang <[email protected]> --- clang/include/clang/Lex/Preprocessor.h | 1 - 1 file changed, 1 deletion(-) diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h index d3633fef9114d..1c917dcfe7b7e 100644 --- a/clang/include/clang/Lex/Preprocessor.h +++ b/clang/include/clang/Lex/Preprocessor.h @@ -49,7 +49,6 @@ #include "llvm/Support/Allocator.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Registry.h" -#include "llvm/Support/SaveAndRestore.h" #include "llvm/Support/TrailingObjects.h" #include <cassert> #include <cstddef> >From 20fdd6770ae1c8a7f467fd969c1f780509c849de Mon Sep 17 00:00:00 2001 From: yronglin <[email protected]> Date: Mon, 27 Jul 2026 09:13:37 -0700 Subject: [PATCH 11/14] [clang] Revert unnecessary changes Signed-off-by: yronglin <[email protected]> --- clang/lib/Lex/Lexer.cpp | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp index 2badd1488daee..eba61b3d76a49 100644 --- a/clang/lib/Lex/Lexer.cpp +++ b/clang/lib/Lex/Lexer.cpp @@ -3287,6 +3287,15 @@ bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) { std::optional<Token> Lexer::peekNextPPToken() { assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?"); + if (isDependencyDirectivesLexer()) { + if (NextDepDirectiveTokenIndex == DepDirectives.front().Tokens.size()) + return std::nullopt; + Token Result; + (void)convertDependencyDirectiveToken( + DepDirectives.front().Tokens[NextDepDirectiveTokenIndex], Result); + return Result; + } + // Switch to 'skipping' mode. This will ensure that we can lex a token // without emitting diagnostics, disables macro expansion, and will cause EOF // to return an EOF token instead of popping the include stack. @@ -3301,14 +3310,7 @@ std::optional<Token> Lexer::peekNextPPToken() { MultipleIncludeOpt MIOptState = MIOpt; Token Tok; - if (isDependencyDirectivesLexer()) { - if (NextDepDirectiveTokenIndex == DepDirectives.front().Tokens.size()) - return std::nullopt; - (void)convertDependencyDirectiveToken( - DepDirectives.front().Tokens[NextDepDirectiveTokenIndex], Tok); - } else { - Lex(Tok); - } + Lex(Tok); // Restore state that may have changed. BufferPtr = TmpBufferPtr; @@ -4680,8 +4682,7 @@ bool Lexer::LexDependencyDirectiveToken(Token &Result) { convertDependencyDirectiveToken(DDTok, Result); return true; } - if (Result.isNot(tok::header_name)) - return true; + // Advance the index of lexed tokens. while (true) { const dependency_directives_scan::Token &NextTok = >From ff381079a82b74dc7c3efb1aa579724b3637c23f Mon Sep 17 00:00:00 2001 From: yronglin <[email protected]> Date: Mon, 3 Aug 2026 09:22:08 -0700 Subject: [PATCH 12/14] Fix Dependency directive scaning Signed-off-by: yronglin <[email protected]> --- clang/lib/Lex/Lexer.cpp | 8 +- .../Lex/PPDependencyDirectivesTest.cpp | 127 ++++++++++++++---- 2 files changed, 106 insertions(+), 29 deletions(-) diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp index eba61b3d76a49..06a4b58eb7727 100644 --- a/clang/lib/Lex/Lexer.cpp +++ b/clang/lib/Lex/Lexer.cpp @@ -4676,8 +4676,12 @@ bool Lexer::LexDependencyDirectiveToken(Token &Result) { MIOpt.ReadToken(); } - if (ParsingFilename && DDTok.is(tok::less)) { - BufferPtr = BufferStart + DDTok.Offset; + const char *DDTokPtr = BufferStart + DDTok.Offset; + if (ParsingFilename && *DDTokPtr == '<') { + Result.startToken(); + Result.setFlag((Token::TokenFlags)DDTok.Flags); + Result.clearFlag(Token::NeedsCleaning); + BufferPtr = DDTokPtr; if (!LexAngledStringLiteral(Result, BufferPtr + 1)) { convertDependencyDirectiveToken(DDTok, Result); return true; diff --git a/clang/unittests/Lex/PPDependencyDirectivesTest.cpp b/clang/unittests/Lex/PPDependencyDirectivesTest.cpp index 6beb66d1a36f5..d987d55023fce 100644 --- a/clang/unittests/Lex/PPDependencyDirectivesTest.cpp +++ b/clang/unittests/Lex/PPDependencyDirectivesTest.cpp @@ -96,6 +96,35 @@ class EmbedCollector : public PPCallbacks { } }; +struct DepDirectives { + SmallVector<dependency_directives_scan::Token> Tokens; + SmallVector<dependency_directives_scan::Directive> Directives; +}; + +class TestDependencyDirectivesGetter : public DependencyDirectivesGetter { + FileManager &FileMgr; + SmallVector<std::unique_ptr<DepDirectives>> DepDirectivesObjects; + +public: + TestDependencyDirectivesGetter(FileManager &FileMgr) : FileMgr(FileMgr) {} + + std::unique_ptr<DependencyDirectivesGetter> + cloneFor(FileManager &FileMgr) override { + return std::make_unique<TestDependencyDirectivesGetter>(FileMgr); + } + + std::optional<ArrayRef<dependency_directives_scan::Directive>> + operator()(FileEntryRef File) override { + DepDirectivesObjects.push_back(std::make_unique<DepDirectives>()); + StringRef Input = (*FileMgr.getBufferForFile(File))->getBuffer(); + bool Err = scanSourceForDependencyDirectives( + Input, DepDirectivesObjects.back()->Tokens, + DepDirectivesObjects.back()->Directives); + EXPECT_FALSE(Err); + return DepDirectivesObjects.back()->Directives; + } +}; + TEST_F(PPDependencyDirectivesTest, MacroGuard) { // "head1.h" has a macro guard and should only be included once. // "head2.h" and "head3.h" have tokens following the macro check, they should @@ -126,45 +155,90 @@ TEST_F(PPDependencyDirectivesTest, MacroGuard) { SourceMgr.setMainFileID( SourceMgr.createFileID(*FE, SourceLocation(), SrcMgr::C_User)); - struct DepDirectives { - SmallVector<dependency_directives_scan::Token> Tokens; - SmallVector<dependency_directives_scan::Directive> Directives; + TestDependencyDirectivesGetter GetDependencyDirectives(FileMgr); + + PreprocessorOptions PPOpts; + HeaderSearchOptions HSOpts; + TrivialModuleLoader ModLoader; + HeaderSearch HeaderInfo(HSOpts, SourceMgr, Diags, LangOpts, Target.get()); + Preprocessor PP(PPOpts, Diags, LangOpts, SourceMgr, HeaderInfo, ModLoader, + /*IILookup =*/nullptr, + /*OwnsHeaderSearch =*/false); + PP.Initialize(*Target); + + PP.setDependencyDirectivesGetter(GetDependencyDirectives); + + SmallVector<StringRef> IncludedFiles; + PP.addPPCallbacks(std::make_unique<IncludeCollector>(PP, IncludedFiles)); + PP.EnterMainSourceFile(); + PP.LexTokensUntilEOF(); + + SmallVector<std::string> IncludedFilesSlash; + for (StringRef IncludedFile : IncludedFiles) + IncludedFilesSlash.push_back( + llvm::sys::path::convert_to_slash(IncludedFile)); + SmallVector<std::string> ExpectedIncludes{ + "main.c", "./head1.h", "./head2.h", "./head2.h", "./head3.h", "./head3.h", }; + EXPECT_EQ(IncludedFilesSlash, ExpectedIncludes); +} - class TestDependencyDirectivesGetter : public DependencyDirectivesGetter { - FileManager &FileMgr; - SmallVector<std::unique_ptr<DepDirectives>> DepDirectivesObjects; +TEST_F(PPDependencyDirectivesTest, HeaderNamesStartingWithCompoundTokens) { + StringRef Source = + "#if __has_include(<=>)\n" + "#include <=>\n" + "#endif\n" + "#if __has_include(<<>)\n" + "#include <<>\n" + "#endif\n"; - public: - TestDependencyDirectivesGetter(FileManager &FileMgr) : FileMgr(FileMgr) {} + SmallVector<dependency_directives_scan::Token> Tokens; + SmallVector<dependency_directives_scan::Directive> Directives; + ASSERT_FALSE(scanSourceForDependencyDirectives(Source, Tokens, Directives)); - std::unique_ptr<DependencyDirectivesGetter> - cloneFor(FileManager &FileMgr) override { - return std::make_unique<TestDependencyDirectivesGetter>(FileMgr); + unsigned LessEqualCount = 0; + unsigned LessLessCount = 0; + for (const dependency_directives_scan::Token &Tok : Tokens) { + StringRef Spelling = Source.slice(Tok.Offset, Tok.getEnd()); + if (Spelling == "<=") { + EXPECT_TRUE(Tok.is(tok::lessequal)); + ++LessEqualCount; + } else if (Spelling == "<<") { + EXPECT_TRUE(Tok.is(tok::lessless)); + ++LessLessCount; } + } + EXPECT_EQ(LessEqualCount, 1u); + EXPECT_EQ(LessLessCount, 1u); - std::optional<ArrayRef<dependency_directives_scan::Directive>> - operator()(FileEntryRef File) override { - DepDirectivesObjects.push_back(std::make_unique<DepDirectives>()); - StringRef Input = (*FileMgr.getBufferForFile(File))->getBuffer(); - bool Err = scanSourceForDependencyDirectives( - Input, DepDirectivesObjects.back()->Tokens, - DepDirectivesObjects.back()->Directives); - EXPECT_FALSE(Err); - return DepDirectivesObjects.back()->Directives; - } - }; - TestDependencyDirectivesGetter GetDependencyDirectives(FileMgr); + auto VFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>(); + VFS->setCurrentWorkingDirectory("/source"); + VFS->addFile("/source/=", 0, llvm::MemoryBuffer::getMemBuffer("")); + VFS->addFile("/source/<", 0, llvm::MemoryBuffer::getMemBuffer("")); + VFS->addFile("/source/main.c", 0, + llvm::MemoryBuffer::getMemBuffer(Source)); + FileMgr.setVirtualFileSystem(VFS); + OptionalFileEntryRef FE; + ASSERT_THAT_ERROR(FileMgr.getFileRef("/source/main.c").moveInto(FE), + llvm::Succeeded()); + SourceMgr.setMainFileID( + SourceMgr.createFileID(*FE, SourceLocation(), SrcMgr::C_User)); + + TestDependencyDirectivesGetter GetDependencyDirectives(FileMgr); PreprocessorOptions PPOpts; HeaderSearchOptions HSOpts; TrivialModuleLoader ModLoader; HeaderSearch HeaderInfo(HSOpts, SourceMgr, Diags, LangOpts, Target.get()); + auto DE = FileMgr.getOptionalDirectoryRef("/source"); + ASSERT_TRUE(DE); + HeaderInfo.AddSearchPath( + DirectoryLookup(*DE, SrcMgr::C_User, /*isFramework=*/false), + /*isAngled=*/true); Preprocessor PP(PPOpts, Diags, LangOpts, SourceMgr, HeaderInfo, ModLoader, /*IILookup =*/nullptr, /*OwnsHeaderSearch =*/false); PP.Initialize(*Target); - PP.setDependencyDirectivesGetter(GetDependencyDirectives); SmallVector<StringRef> IncludedFiles; @@ -176,9 +250,8 @@ TEST_F(PPDependencyDirectivesTest, MacroGuard) { for (StringRef IncludedFile : IncludedFiles) IncludedFilesSlash.push_back( llvm::sys::path::convert_to_slash(IncludedFile)); - SmallVector<std::string> ExpectedIncludes{ - "main.c", "./head1.h", "./head2.h", "./head2.h", "./head3.h", "./head3.h", - }; + SmallVector<std::string> ExpectedIncludes{"/source/main.c", "/source/=", + "/source/<"}; EXPECT_EQ(IncludedFilesSlash, ExpectedIncludes); } >From 9583cab30407fe74291989daaf3cffd2f6ca7053 Mon Sep 17 00:00:00 2001 From: yronglin <[email protected]> Date: Tue, 4 Aug 2026 01:11:38 -0700 Subject: [PATCH 13/14] Add fixme and more test Signed-off-by: yronglin <[email protected]> --- clang/lib/Lex/Lexer.cpp | 2 ++ clang/lib/Lex/Preprocessor.cpp | 3 ++- .../Lex/PPDependencyDirectivesTest.cpp | 21 +++++++++---------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp index 6eefb58740d29..31a742b56f0f7 100644 --- a/clang/lib/Lex/Lexer.cpp +++ b/clang/lib/Lex/Lexer.cpp @@ -4708,6 +4708,8 @@ bool Lexer::LexDependencyDirectiveToken(Token &Result) { } // Advance the index of lexed tokens. + // FIXME: This will skip too many tokens if the header-name ended in the + // middle of a token, such as in '<foo>='. while (true) { const dependency_directives_scan::Token &NextTok = DepDirectives.front().Tokens[NextDepDirectiveTokenIndex]; diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp index b3928fe0cf670..62d67e08e6083 100644 --- a/clang/lib/Lex/Preprocessor.cpp +++ b/clang/lib/Lex/Preprocessor.cpp @@ -1379,7 +1379,8 @@ bool Preprocessor::HandleModuleContextualKeyword(Token &Result) { if (auto NextTok = peekNextPPToken()) { if (NextTok->is(tok::raw_identifier)) LookUpIdentifierInfo(*NextTok); - if (NextTok->isOneOf(tok::header_name, tok::identifier, tok::colon, tok::less)) { + if (NextTok->isOneOf(tok::header_name, tok::identifier, tok::colon, + tok::less)) { Result.setKind(tok::kw_import); ModuleImportLoc = Result.getLocation(); return true; diff --git a/clang/unittests/Lex/PPDependencyDirectivesTest.cpp b/clang/unittests/Lex/PPDependencyDirectivesTest.cpp index d987d55023fce..3b369ffc60d27 100644 --- a/clang/unittests/Lex/PPDependencyDirectivesTest.cpp +++ b/clang/unittests/Lex/PPDependencyDirectivesTest.cpp @@ -184,13 +184,13 @@ TEST_F(PPDependencyDirectivesTest, MacroGuard) { } TEST_F(PPDependencyDirectivesTest, HeaderNamesStartingWithCompoundTokens) { - StringRef Source = - "#if __has_include(<=>)\n" - "#include <=>\n" - "#endif\n" - "#if __has_include(<<>)\n" - "#include <<>\n" - "#endif\n"; + StringRef Source = "#if __has_include(<=>)\n" + "#include <=>\n" + "#endif\n" + "#if __has_include(<<>)\n" + "#include <<>\n" + "#endif\n" + "#include \\\n<=>"; SmallVector<dependency_directives_scan::Token> Tokens; SmallVector<dependency_directives_scan::Directive> Directives; @@ -215,8 +215,7 @@ TEST_F(PPDependencyDirectivesTest, HeaderNamesStartingWithCompoundTokens) { VFS->setCurrentWorkingDirectory("/source"); VFS->addFile("/source/=", 0, llvm::MemoryBuffer::getMemBuffer("")); VFS->addFile("/source/<", 0, llvm::MemoryBuffer::getMemBuffer("")); - VFS->addFile("/source/main.c", 0, - llvm::MemoryBuffer::getMemBuffer(Source)); + VFS->addFile("/source/main.c", 0, llvm::MemoryBuffer::getMemBuffer(Source)); FileMgr.setVirtualFileSystem(VFS); OptionalFileEntryRef FE; @@ -250,8 +249,8 @@ TEST_F(PPDependencyDirectivesTest, HeaderNamesStartingWithCompoundTokens) { for (StringRef IncludedFile : IncludedFiles) IncludedFilesSlash.push_back( llvm::sys::path::convert_to_slash(IncludedFile)); - SmallVector<std::string> ExpectedIncludes{"/source/main.c", "/source/=", - "/source/<"}; + SmallVector<std::string> ExpectedIncludes{ + "/source/main.c", "/source/=", "/source/<", "/source/="}; EXPECT_EQ(IncludedFilesSlash, ExpectedIncludes); } >From 6d38aba3a1f5869c75e3acd5995a2dd5892e8d48 Mon Sep 17 00:00:00 2001 From: yronglin <[email protected]> Date: Tue, 4 Aug 2026 02:23:20 -0700 Subject: [PATCH 14/14] Format Signed-off-by: yronglin <[email protected]> --- clang/lib/Lex/Lexer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp index 31a742b56f0f7..6530c0606d10e 100644 --- a/clang/lib/Lex/Lexer.cpp +++ b/clang/lib/Lex/Lexer.cpp @@ -4394,7 +4394,7 @@ bool Lexer::LexTokenInternal(Token &Result) { } CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); Kind = tok::lessequal; - } else if (LangOpts.Digraphs && Char == ':') { // '<:' -> '[' + } else if (LangOpts.Digraphs && Char == ':') { // '<:' -> '[' if (LangOpts.CPlusPlus11 && getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == ':') { // C++0x [lex.pptoken]p3: @@ -4414,7 +4414,7 @@ bool Lexer::LexTokenInternal(Token &Result) { CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); Kind = tok::l_square; - } else if (LangOpts.Digraphs && Char == '%') { // '<%' -> '{' + } else if (LangOpts.Digraphs && Char == '%') { // '<%' -> '{' CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); Kind = tok::l_brace; } else if (Char == '#' && /*Not a trigraph*/ SizeTmp == 1 && _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
