usaxena95 created this revision.
usaxena95 added a reviewer: ilya-biryukov.
Herald added a project: clang.
Herald added a subscriber: cfe-commits.
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D68565
Files:
clang/unittests/Lex/LexerTest.cpp
Index: clang/unittests/Lex/LexerTest.cpp
===================================================================
--- clang/unittests/Lex/LexerTest.cpp
+++ clang/unittests/Lex/LexerTest.cpp
@@ -11,9 +11,11 @@
#include "clang/Basic/DiagnosticOptions.h"
#include "clang/Basic/FileManager.h"
#include "clang/Basic/LangOptions.h"
+#include "clang/Basic/SourceLocation.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Basic/TargetInfo.h"
#include "clang/Basic/TargetOptions.h"
+#include "clang/Basic/TokenKinds.h"
#include "clang/Lex/HeaderSearch.h"
#include "clang/Lex/HeaderSearchOptions.h"
#include "clang/Lex/MacroArgs.h"
@@ -22,6 +24,7 @@
#include "clang/Lex/Preprocessor.h"
#include "clang/Lex/PreprocessorOptions.h"
#include "gtest/gtest.h"
+#include <vector>
using namespace clang;
@@ -31,12 +34,9 @@
class LexerTest : public ::testing::Test {
protected:
LexerTest()
- : FileMgr(FileMgrOpts),
- DiagID(new DiagnosticIDs()),
- Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
- SourceMgr(Diags, FileMgr),
- TargetOpts(new TargetOptions)
- {
+ : FileMgr(FileMgrOpts), DiagID(new DiagnosticIDs()),
+ Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
+ SourceMgr(Diags, FileMgr), TargetOpts(new TargetOptions) {
TargetOpts->Triple = "x86_64-apple-darwin11.1.0";
Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts);
}
@@ -90,7 +90,7 @@
bool Invalid;
StringRef Str =
Lexer::getSourceText(CharSourceRange::getTokenRange(SourceRange(
- Begin.getLocation(), End.getLocation())),
+ Begin.getLocation(), End.getLocation())),
SourceMgr, LangOpts, &Invalid);
if (Invalid)
return "<INVALID>";
@@ -286,30 +286,31 @@
CharSourceRange macroRange = SourceMgr.getExpansionRange(lsqrLoc);
SourceLocation Loc;
- EXPECT_TRUE(Lexer::isAtStartOfMacroExpansion(lsqrLoc, SourceMgr, LangOpts, &Loc));
+ EXPECT_TRUE(
+ Lexer::isAtStartOfMacroExpansion(lsqrLoc, SourceMgr, LangOpts, &Loc));
EXPECT_EQ(Loc, macroRange.getBegin());
EXPECT_FALSE(Lexer::isAtStartOfMacroExpansion(idLoc, SourceMgr, LangOpts));
EXPECT_FALSE(Lexer::isAtEndOfMacroExpansion(idLoc, SourceMgr, LangOpts));
- EXPECT_TRUE(Lexer::isAtEndOfMacroExpansion(rsqrLoc, SourceMgr, LangOpts, &Loc));
+ EXPECT_TRUE(
+ Lexer::isAtEndOfMacroExpansion(rsqrLoc, SourceMgr, LangOpts, &Loc));
EXPECT_EQ(Loc, macroRange.getEnd());
EXPECT_TRUE(macroRange.isTokenRange());
CharSourceRange range = Lexer::makeFileCharRange(
- CharSourceRange::getTokenRange(lsqrLoc, idLoc), SourceMgr, LangOpts);
+ CharSourceRange::getTokenRange(lsqrLoc, idLoc), SourceMgr, LangOpts);
EXPECT_TRUE(range.isInvalid());
- range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(idLoc, rsqrLoc),
- SourceMgr, LangOpts);
+ range = Lexer::makeFileCharRange(
+ CharSourceRange::getTokenRange(idLoc, rsqrLoc), SourceMgr, LangOpts);
EXPECT_TRUE(range.isInvalid());
- range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
- SourceMgr, LangOpts);
+ range = Lexer::makeFileCharRange(
+ CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc), SourceMgr, LangOpts);
EXPECT_TRUE(!range.isTokenRange());
EXPECT_EQ(range.getAsRange(),
SourceRange(macroRange.getBegin(),
macroRange.getEnd().getLocWithOffset(1)));
StringRef text = Lexer::getSourceText(
- CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
- SourceMgr, LangOpts);
+ CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc), SourceMgr, LangOpts);
EXPECT_EQ(text, "M(foo)");
SourceLocation macroLsqrLoc = toks[3].getLocation();
@@ -320,29 +321,30 @@
SourceLocation fileRsqrLoc = SourceMgr.getSpellingLoc(macroRsqrLoc);
range = Lexer::makeFileCharRange(
- CharSourceRange::getTokenRange(macroLsqrLoc, macroIdLoc),
- SourceMgr, LangOpts);
+ CharSourceRange::getTokenRange(macroLsqrLoc, macroIdLoc), SourceMgr,
+ LangOpts);
EXPECT_EQ(SourceRange(fileLsqrLoc, fileIdLoc.getLocWithOffset(3)),
range.getAsRange());
- range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(macroIdLoc, macroRsqrLoc),
- SourceMgr, LangOpts);
+ range = Lexer::makeFileCharRange(
+ CharSourceRange::getTokenRange(macroIdLoc, macroRsqrLoc), SourceMgr,
+ LangOpts);
EXPECT_EQ(SourceRange(fileIdLoc, fileRsqrLoc.getLocWithOffset(1)),
range.getAsRange());
macroRange = SourceMgr.getExpansionRange(macroLsqrLoc);
range = Lexer::makeFileCharRange(
- CharSourceRange::getTokenRange(macroLsqrLoc, macroRsqrLoc),
- SourceMgr, LangOpts);
- EXPECT_EQ(SourceRange(macroRange.getBegin(), macroRange.getEnd().getLocWithOffset(1)),
+ CharSourceRange::getTokenRange(macroLsqrLoc, macroRsqrLoc), SourceMgr,
+ LangOpts);
+ EXPECT_EQ(SourceRange(macroRange.getBegin(),
+ macroRange.getEnd().getLocWithOffset(1)),
range.getAsRange());
text = Lexer::getSourceText(
- CharSourceRange::getTokenRange(SourceRange(macroLsqrLoc, macroIdLoc)),
- SourceMgr, LangOpts);
+ CharSourceRange::getTokenRange(SourceRange(macroLsqrLoc, macroIdLoc)),
+ SourceMgr, LangOpts);
EXPECT_EQ(text, "[bar");
-
SourceLocation idLoc1 = toks[6].getLocation();
SourceLocation idLoc2 = toks[7].getLocation();
SourceLocation idLoc3 = toks[8].getLocation();
@@ -527,12 +529,29 @@
EXPECT_TRUE(
Lexer::isAtStartOfMacroExpansion(R.getBegin(), SourceMgr, LangOpts));
- EXPECT_TRUE(
- Lexer::isAtEndOfMacroExpansion(R.getEnd(), SourceMgr, LangOpts));
+ EXPECT_TRUE(Lexer::isAtEndOfMacroExpansion(R.getEnd(), SourceMgr, LangOpts));
CharSourceRange CR = Lexer::getAsCharRange(R, SourceMgr, LangOpts);
EXPECT_EQ(Lexer::getSourceText(CR, SourceMgr, LangOpts), "MOO"); // Was "MO".
}
+TEST_F(LexerTest, FindNextToken) {
+ Lex("int abcd = 0;\n"
+ "int xyz = abcd;\n");
+ std::vector<std::string> ExpectedTokens = {"abcd", "=", "0", ";", "int",
+ "xyz", "=", "abcd", ";"};
+ std::vector<std::string> GeneratedByNextToken;
+ SourceLocation Loc =
+ SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID());
+ while (true) {
+ auto T = Lexer::findNextToken(Loc, SourceMgr, LangOpts);
+ ASSERT_TRUE(T.hasValue());
+ if (T->is(tok::eof))
+ break;
+ GeneratedByNextToken.push_back(getSourceText(*T, *T));
+ Loc = T->getLocation();
+ }
+ EXPECT_EQ(ExpectedTokens, GeneratedByNextToken);
+}
} // anonymous namespace
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
